2024-11-11 13:34:55,855 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-11 13:34:55,866 main DEBUG Took 0.009863 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 13:34:55,867 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 13:34:55,867 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 13:34:55,868 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 13:34:55,869 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,882 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 13:34:55,893 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,894 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,895 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,895 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,895 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,896 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,897 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,897 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,897 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,898 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,899 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,899 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,899 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,900 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,900 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,901 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,901 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,902 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,902 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,903 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:34:55,903 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,904 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 13:34:55,905 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:34:55,906 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 13:34:55,908 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 13:34:55,908 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 13:34:55,909 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 13:34:55,910 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 13:34:55,917 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 13:34:55,919 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 13:34:55,921 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 13:34:55,922 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 13:34:55,922 main DEBUG createAppenders(={Console}) 2024-11-11 13:34:55,923 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-11 13:34:55,923 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-11 13:34:55,923 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-11 13:34:55,924 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 13:34:55,924 main DEBUG OutputStream closed 2024-11-11 13:34:55,924 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 13:34:55,925 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 13:34:55,925 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4b0d79fc OK 2024-11-11 13:34:55,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 13:34:55,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 13:34:55,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 13:34:55,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 13:34:55,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 13:34:55,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 13:34:55,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 13:34:55,999 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 13:34:55,999 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 13:34:55,999 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 13:34:56,000 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 13:34:56,000 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 13:34:56,000 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 13:34:56,000 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 13:34:56,001 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 13:34:56,001 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 13:34:56,001 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 13:34:56,002 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 13:34:56,004 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 13:34:56,005 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@68b32e3e) with optional ClassLoader: null 2024-11-11 13:34:56,005 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 13:34:56,006 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@68b32e3e] started OK. 2024-11-11T13:34:56,260 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813 2024-11-11 13:34:56,263 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 13:34:56,263 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T13:34:56,270 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-11T13:34:56,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T13:34:56,305 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/cluster_53e0cb5c-a2a6-7eb1-6f1f-43fb1df4ddb1, deleteOnExit=true 2024-11-11T13:34:56,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T13:34:56,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/test.cache.data in system properties and HBase conf 2024-11-11T13:34:56,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T13:34:56,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/hadoop.log.dir in system properties and HBase conf 2024-11-11T13:34:56,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T13:34:56,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T13:34:56,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T13:34:56,388 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T13:34:56,474 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:34:56,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T13:34:56,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T13:34:56,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T13:34:56,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T13:34:56,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T13:34:56,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T13:34:56,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T13:34:56,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T13:34:56,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T13:34:56,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/nfs.dump.dir in system properties and HBase conf 2024-11-11T13:34:56,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/java.io.tmpdir in system properties and HBase conf 2024-11-11T13:34:56,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T13:34:56,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T13:34:56,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T13:34:57,425 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T13:34:57,494 INFO [Time-limited test {}] log.Log(170): Logging initialized @2553ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T13:34:57,559 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T13:34:57,615 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T13:34:57,635 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T13:34:57,636 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T13:34:57,637 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T13:34:57,648 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T13:34:57,650 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f9fd429{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/hadoop.log.dir/,AVAILABLE} 2024-11-11T13:34:57,651 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40029c9a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T13:34:57,835 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a5ffc6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/java.io.tmpdir/jetty-localhost-46821-hadoop-hdfs-3_4_1-tests_jar-_-any-1306685323798612565/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T13:34:57,842 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f30a77e{HTTP/1.1, (http/1.1)}{localhost:46821} 2024-11-11T13:34:57,842 INFO [Time-limited test {}] server.Server(415): Started @2901ms 2024-11-11T13:34:58,398 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T13:34:58,405 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T13:34:58,406 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T13:34:58,406 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T13:34:58,406 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T13:34:58,407 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d058786{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/hadoop.log.dir/,AVAILABLE} 2024-11-11T13:34:58,407 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@50fd64ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T13:34:58,516 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@58e0e813{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/java.io.tmpdir/jetty-localhost-44637-hadoop-hdfs-3_4_1-tests_jar-_-any-1364389995278903473/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T13:34:58,517 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@716e8981{HTTP/1.1, (http/1.1)}{localhost:44637} 2024-11-11T13:34:58,517 INFO [Time-limited test {}] server.Server(415): Started @3576ms 2024-11-11T13:34:58,572 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T13:34:59,552 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/cluster_53e0cb5c-a2a6-7eb1-6f1f-43fb1df4ddb1/data/data1/current/BP-476254492-172.17.0.3-1731332096997/current, will proceed with Du for space computation calculation, 2024-11-11T13:34:59,552 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/cluster_53e0cb5c-a2a6-7eb1-6f1f-43fb1df4ddb1/data/data2/current/BP-476254492-172.17.0.3-1731332096997/current, will proceed with Du for space computation calculation, 2024-11-11T13:34:59,603 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T13:34:59,664 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3d895c7eac0e6438 with lease ID 0x4bfb7b44b740504e: Processing first storage report for DS-2d9505b7-f554-48a9-be31-f94be472e7f8 from datanode DatanodeRegistration(127.0.0.1:46037, datanodeUuid=077efd71-23ba-4051-bb1b-fe9a3773ce51, infoPort=42733, infoSecurePort=0, ipcPort=46039, storageInfo=lv=-57;cid=testClusterID;nsid=358737182;c=1731332096997) 2024-11-11T13:34:59,666 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3d895c7eac0e6438 with lease ID 0x4bfb7b44b740504e: from storage DS-2d9505b7-f554-48a9-be31-f94be472e7f8 node DatanodeRegistration(127.0.0.1:46037, datanodeUuid=077efd71-23ba-4051-bb1b-fe9a3773ce51, infoPort=42733, infoSecurePort=0, ipcPort=46039, storageInfo=lv=-57;cid=testClusterID;nsid=358737182;c=1731332096997), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-11T13:34:59,667 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3d895c7eac0e6438 with lease ID 0x4bfb7b44b740504e: Processing first storage report for DS-410f57fb-bb6b-4faa-908b-24ca8a155fbe from datanode DatanodeRegistration(127.0.0.1:46037, datanodeUuid=077efd71-23ba-4051-bb1b-fe9a3773ce51, infoPort=42733, infoSecurePort=0, ipcPort=46039, storageInfo=lv=-57;cid=testClusterID;nsid=358737182;c=1731332096997) 2024-11-11T13:34:59,667 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3d895c7eac0e6438 with lease ID 0x4bfb7b44b740504e: from storage DS-410f57fb-bb6b-4faa-908b-24ca8a155fbe node DatanodeRegistration(127.0.0.1:46037, datanodeUuid=077efd71-23ba-4051-bb1b-fe9a3773ce51, infoPort=42733, infoSecurePort=0, ipcPort=46039, storageInfo=lv=-57;cid=testClusterID;nsid=358737182;c=1731332096997), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T13:34:59,758 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813 2024-11-11T13:34:59,841 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/cluster_53e0cb5c-a2a6-7eb1-6f1f-43fb1df4ddb1/zookeeper_0, clientPort=52686, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/cluster_53e0cb5c-a2a6-7eb1-6f1f-43fb1df4ddb1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/cluster_53e0cb5c-a2a6-7eb1-6f1f-43fb1df4ddb1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T13:34:59,850 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52686 2024-11-11T13:34:59,860 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T13:34:59,862 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T13:35:00,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741825_1001 (size=7) 2024-11-11T13:35:00,493 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92 with version=8 2024-11-11T13:35:00,494 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/hbase-staging 2024-11-11T13:35:00,624 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T13:35:00,892 INFO [Time-limited test {}] client.ConnectionUtils(128): master/37d26ecf6460:0 server-side Connection retries=45 2024-11-11T13:35:00,904 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:35:00,905 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T13:35:00,910 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T13:35:00,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:35:00,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T13:35:01,089 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T13:35:01,165 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T13:35:01,174 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T13:35:01,178 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T13:35:01,202 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 38284 (auto-detected) 2024-11-11T13:35:01,203 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-11T13:35:01,224 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35609 2024-11-11T13:35:01,250 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35609 connecting to ZooKeeper ensemble=127.0.0.1:52686 2024-11-11T13:35:01,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:356090x0, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T13:35:01,344 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35609-0x1012b6500d60000 connected 2024-11-11T13:35:01,432 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T13:35:01,436 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T13:35:01,448 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:35:01,454 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92, hbase.cluster.distributed=false 2024-11-11T13:35:01,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T13:35:01,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35609 2024-11-11T13:35:01,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35609 2024-11-11T13:35:01,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35609 2024-11-11T13:35:01,515 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35609 2024-11-11T13:35:01,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35609 2024-11-11T13:35:01,606 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/37d26ecf6460:0 server-side Connection retries=45 2024-11-11T13:35:01,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:35:01,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T13:35:01,608 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T13:35:01,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:35:01,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T13:35:01,611 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T13:35:01,613 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T13:35:01,614 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36877 2024-11-11T13:35:01,616 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36877 connecting to ZooKeeper ensemble=127.0.0.1:52686 2024-11-11T13:35:01,617 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T13:35:01,620 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T13:35:01,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368770x0, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T13:35:01,631 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36877-0x1012b6500d60001 connected 2024-11-11T13:35:01,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:368770x0, quorum=127.0.0.1:52686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:35:01,636 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T13:35:01,646 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T13:35:01,649 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T13:35:01,653 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T13:35:01,654 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36877 2024-11-11T13:35:01,654 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36877 2024-11-11T13:35:01,655 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36877 2024-11-11T13:35:01,659 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36877 2024-11-11T13:35:01,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36877 2024-11-11T13:35:01,674 DEBUG [M:0;37d26ecf6460:35609 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;37d26ecf6460:35609 2024-11-11T13:35:01,675 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/37d26ecf6460,35609,1731332100707 2024-11-11T13:35:01,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:35:01,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:35:01,690 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/37d26ecf6460,35609,1731332100707 2024-11-11T13:35:01,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T13:35:01,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:01,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:01,714 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T13:35:01,715 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/37d26ecf6460,35609,1731332100707 from backup master directory 2024-11-11T13:35:01,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:35:01,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/37d26ecf6460,35609,1731332100707 2024-11-11T13:35:01,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:35:01,722 WARN [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T13:35:01,722 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=37d26ecf6460,35609,1731332100707 2024-11-11T13:35:01,724 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T13:35:01,725 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T13:35:01,780 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/hbase.id] with ID: 7f38533c-3e78-4919-855f-edcf552b3d40 2024-11-11T13:35:01,780 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/.tmp/hbase.id 2024-11-11T13:35:01,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741826_1002 (size=42) 2024-11-11T13:35:01,790 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/.tmp/hbase.id]:[hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/hbase.id] 2024-11-11T13:35:01,836 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T13:35:01,841 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T13:35:01,860 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-11T13:35:01,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:01,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:01,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741827_1003 (size=196) 2024-11-11T13:35:01,925 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:35:01,928 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T13:35:01,944 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:01,948 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:35:01,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741828_1004 (size=1189) 2024-11-11T13:35:02,409 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store 2024-11-11T13:35:02,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741829_1005 (size=34) 2024-11-11T13:35:02,832 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T13:35:02,836 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:35:02,838 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T13:35:02,838 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:35:02,838 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:35:02,840 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T13:35:02,840 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:35:02,841 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:35:02,842 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731332102837Disabling compacts and flushes for region at 1731332102837Disabling writes for close at 1731332102840 (+3 ms)Writing region close event to WAL at 1731332102840Closed at 1731332102840 2024-11-11T13:35:02,845 WARN [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/.initializing 2024-11-11T13:35:02,846 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/WALs/37d26ecf6460,35609,1731332100707 2024-11-11T13:35:02,855 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T13:35:02,871 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=37d26ecf6460%2C35609%2C1731332100707, suffix=, logDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/WALs/37d26ecf6460,35609,1731332100707, archiveDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/oldWALs, maxLogs=10 2024-11-11T13:35:02,897 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/WALs/37d26ecf6460,35609,1731332100707/37d26ecf6460%2C35609%2C1731332100707.1731332102876, exclude list is [], retry=0 2024-11-11T13:35:02,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46037,DS-2d9505b7-f554-48a9-be31-f94be472e7f8,DISK] 2024-11-11T13:35:02,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T13:35:02,956 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/WALs/37d26ecf6460,35609,1731332100707/37d26ecf6460%2C35609%2C1731332100707.1731332102876 2024-11-11T13:35:02,957 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42733:42733)] 2024-11-11T13:35:02,957 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:35:02,958 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:35:02,961 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:02,962 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:02,997 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:03,029 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T13:35:03,032 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:03,035 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:35:03,035 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:03,038 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T13:35:03,039 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:03,040 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:03,040 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:03,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T13:35:03,043 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:03,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:03,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:03,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T13:35:03,047 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:03,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:03,048 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:03,052 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:03,053 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:03,058 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:03,058 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:03,063 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T13:35:03,069 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:35:03,074 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:35:03,076 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71747382, jitterRate=0.06911930441856384}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T13:35:03,082 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731332102974Initializing all the Stores at 1731332102976 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731332102977 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332102978 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332102978Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332102978Cleaning up temporary data from old regions at 1731332103059 (+81 ms)Region opened successfully at 1731332103082 (+23 ms) 2024-11-11T13:35:03,084 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T13:35:03,119 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f113b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37d26ecf6460/172.17.0.3:0 2024-11-11T13:35:03,148 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T13:35:03,162 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T13:35:03,162 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T13:35:03,166 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T13:35:03,168 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T13:35:03,175 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-11T13:35:03,176 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T13:35:03,214 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T13:35:03,228 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T13:35:03,263 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T13:35:03,267 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T13:35:03,269 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T13:35:03,280 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T13:35:03,284 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T13:35:03,289 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T13:35:03,296 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T13:35:03,299 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T13:35:03,305 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T13:35:03,331 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T13:35:03,338 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T13:35:03,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T13:35:03,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T13:35:03,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:03,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:03,349 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=37d26ecf6460,35609,1731332100707, sessionid=0x1012b6500d60000, setting cluster-up flag (Was=false) 2024-11-11T13:35:03,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:03,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:03,396 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T13:35:03,399 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=37d26ecf6460,35609,1731332100707 2024-11-11T13:35:03,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:03,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:03,446 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T13:35:03,449 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=37d26ecf6460,35609,1731332100707 2024-11-11T13:35:03,457 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T13:35:03,467 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(746): ClusterId : 7f38533c-3e78-4919-855f-edcf552b3d40 2024-11-11T13:35:03,469 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T13:35:03,482 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T13:35:03,482 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T13:35:03,490 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T13:35:03,490 DEBUG [RS:0;37d26ecf6460:36877 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@576e47e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37d26ecf6460/172.17.0.3:0 2024-11-11T13:35:03,504 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;37d26ecf6460:36877 2024-11-11T13:35:03,507 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T13:35:03,507 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T13:35:03,507 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T13:35:03,510 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(2659): reportForDuty to master=37d26ecf6460,35609,1731332100707 with port=36877, startcode=1731332101576 2024-11-11T13:35:03,520 DEBUG [RS:0;37d26ecf6460:36877 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T13:35:03,533 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T13:35:03,541 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T13:35:03,548 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49225, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T13:35:03,548 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T13:35:03,553 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35609 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:03,553 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 37d26ecf6460,35609,1731332100707 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T13:35:03,560 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:35:03,560 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:35:03,560 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:35:03,561 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:35:03,561 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/37d26ecf6460:0, corePoolSize=10, maxPoolSize=10 2024-11-11T13:35:03,561 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,561 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/37d26ecf6460:0, corePoolSize=2, maxPoolSize=2 2024-11-11T13:35:03,561 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,563 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731332133563 2024-11-11T13:35:03,564 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T13:35:03,566 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T13:35:03,566 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T13:35:03,567 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T13:35:03,569 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T13:35:03,569 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T13:35:03,570 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T13:35:03,570 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T13:35:03,571 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,574 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T13:35:03,574 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:03,574 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T13:35:03,575 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T13:35:03,576 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T13:35:03,578 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T13:35:03,578 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T13:35:03,583 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-11T13:35:03,583 WARN [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-11T13:35:03,583 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.large.0-1731332103579,5,FailOnTimeoutGroup] 2024-11-11T13:35:03,584 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.small.0-1731332103583,5,FailOnTimeoutGroup] 2024-11-11T13:35:03,584 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,584 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T13:35:03,585 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,586 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741831_1007 (size=1321) 2024-11-11T13:35:03,685 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(2659): reportForDuty to master=37d26ecf6460,35609,1731332100707 with port=36877, startcode=1731332101576 2024-11-11T13:35:03,688 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35609 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 37d26ecf6460,36877,1731332101576 2024-11-11T13:35:03,692 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35609 {}] master.ServerManager(517): Registering regionserver=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:03,701 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92 2024-11-11T13:35:03,701 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36899 2024-11-11T13:35:03,701 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T13:35:03,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T13:35:03,733 DEBUG [RS:0;37d26ecf6460:36877 {}] zookeeper.ZKUtil(111): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/37d26ecf6460,36877,1731332101576 2024-11-11T13:35:03,733 WARN [RS:0;37d26ecf6460:36877 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T13:35:03,734 INFO [RS:0;37d26ecf6460:36877 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:35:03,734 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/WALs/37d26ecf6460,36877,1731332101576 2024-11-11T13:35:03,735 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [37d26ecf6460,36877,1731332101576] 2024-11-11T13:35:03,759 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T13:35:03,775 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T13:35:03,780 INFO [RS:0;37d26ecf6460:36877 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T13:35:03,780 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,781 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T13:35:03,787 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T13:35:03,788 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,788 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,789 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,789 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,789 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,789 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,789 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/37d26ecf6460:0, corePoolSize=2, maxPoolSize=2 2024-11-11T13:35:03,789 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,790 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,790 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,790 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,790 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,790 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:35:03,790 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:35:03,790 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:35:03,794 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,795 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,795 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,795 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,795 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,795 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,36877,1731332101576-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T13:35:03,817 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T13:35:03,819 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,36877,1731332101576-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,819 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,820 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.Replication(171): 37d26ecf6460,36877,1731332101576 started 2024-11-11T13:35:03,841 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:03,841 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1482): Serving as 37d26ecf6460,36877,1731332101576, RpcServer on 37d26ecf6460/172.17.0.3:36877, sessionid=0x1012b6500d60001 2024-11-11T13:35:03,842 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T13:35:03,842 DEBUG [RS:0;37d26ecf6460:36877 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 37d26ecf6460,36877,1731332101576 2024-11-11T13:35:03,843 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,36877,1731332101576' 2024-11-11T13:35:03,843 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T13:35:03,844 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T13:35:03,845 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T13:35:03,845 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T13:35:03,845 DEBUG [RS:0;37d26ecf6460:36877 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 37d26ecf6460,36877,1731332101576 2024-11-11T13:35:03,846 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,36877,1731332101576' 2024-11-11T13:35:03,846 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T13:35:03,847 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T13:35:03,847 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T13:35:03,848 INFO [RS:0;37d26ecf6460:36877 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T13:35:03,848 INFO [RS:0;37d26ecf6460:36877 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T13:35:03,953 INFO [RS:0;37d26ecf6460:36877 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T13:35:03,957 INFO [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=37d26ecf6460%2C36877%2C1731332101576, suffix=, logDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/WALs/37d26ecf6460,36877,1731332101576, archiveDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/oldWALs, maxLogs=32 2024-11-11T13:35:03,983 DEBUG [RS:0;37d26ecf6460:36877 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/WALs/37d26ecf6460,36877,1731332101576/37d26ecf6460%2C36877%2C1731332101576.1731332103962, exclude list is [], retry=0 2024-11-11T13:35:03,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46037,DS-2d9505b7-f554-48a9-be31-f94be472e7f8,DISK] 2024-11-11T13:35:03,989 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T13:35:03,990 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92 2024-11-11T13:35:03,994 INFO [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/WALs/37d26ecf6460,36877,1731332101576/37d26ecf6460%2C36877%2C1731332101576.1731332103962 2024-11-11T13:35:03,995 DEBUG [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42733:42733)] 2024-11-11T13:35:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741833_1009 (size=32) 2024-11-11T13:35:04,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:35:04,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T13:35:04,416 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T13:35:04,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:04,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:35:04,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T13:35:04,421 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T13:35:04,421 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:04,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:35:04,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T13:35:04,429 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T13:35:04,429 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:04,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:35:04,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T13:35:04,434 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T13:35:04,434 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:04,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:35:04,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T13:35:04,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740 2024-11-11T13:35:04,438 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740 2024-11-11T13:35:04,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T13:35:04,441 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T13:35:04,441 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T13:35:04,444 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T13:35:04,448 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:35:04,449 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72990914, jitterRate=0.0876493752002716}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T13:35:04,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731332104409Initializing all the Stores at 1731332104412 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731332104412Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731332104412Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332104412Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731332104412Cleaning up temporary data from old regions at 1731332104441 (+29 ms)Region opened successfully at 1731332104452 (+11 ms) 2024-11-11T13:35:04,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T13:35:04,452 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T13:35:04,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T13:35:04,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T13:35:04,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T13:35:04,454 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T13:35:04,454 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731332104452Disabling compacts and flushes for region at 1731332104452Disabling writes for close at 1731332104452Writing region close event to WAL at 1731332104454 (+2 ms)Closed at 1731332104454 2024-11-11T13:35:04,458 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T13:35:04,458 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T13:35:04,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T13:35:04,475 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T13:35:04,478 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T13:35:04,632 DEBUG [37d26ecf6460:35609 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T13:35:04,647 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:04,654 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 37d26ecf6460,36877,1731332101576, state=OPENING 2024-11-11T13:35:04,696 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T13:35:04,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:04,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:35:04,706 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:35:04,706 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:35:04,708 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T13:35:04,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:35:04,893 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T13:35:04,898 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38611, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T13:35:04,913 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T13:35:04,914 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:35:04,915 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T13:35:04,919 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=37d26ecf6460%2C36877%2C1731332101576.meta, suffix=.meta, logDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/WALs/37d26ecf6460,36877,1731332101576, archiveDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/oldWALs, maxLogs=32 2024-11-11T13:35:04,936 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/WALs/37d26ecf6460,36877,1731332101576/37d26ecf6460%2C36877%2C1731332101576.meta.1731332104921.meta, exclude list is [], retry=0 2024-11-11T13:35:04,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46037,DS-2d9505b7-f554-48a9-be31-f94be472e7f8,DISK] 2024-11-11T13:35:04,946 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/WALs/37d26ecf6460,36877,1731332101576/37d26ecf6460%2C36877%2C1731332101576.meta.1731332104921.meta 2024-11-11T13:35:04,947 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42733:42733)] 2024-11-11T13:35:04,947 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:35:04,950 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T13:35:04,953 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T13:35:04,959 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T13:35:04,964 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T13:35:04,965 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:35:04,965 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T13:35:04,965 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T13:35:04,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T13:35:04,971 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T13:35:04,971 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:04,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:35:04,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T13:35:04,975 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T13:35:04,975 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:04,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:35:04,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T13:35:04,978 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T13:35:04,978 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:04,979 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:35:04,980 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T13:35:04,981 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T13:35:04,981 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:04,982 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:35:04,983 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T13:35:04,984 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740 2024-11-11T13:35:04,989 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740 2024-11-11T13:35:04,992 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T13:35:04,993 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T13:35:04,994 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T13:35:04,997 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T13:35:04,999 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66073237, jitterRate=-0.015432044863700867}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T13:35:04,999 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T13:35:05,000 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731332104965Writing region info on filesystem at 1731332104966 (+1 ms)Initializing all the Stores at 1731332104968 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731332104968Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731332104969 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332104969Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731332104969Cleaning up temporary data from old regions at 1731332104993 (+24 ms)Running coprocessor post-open hooks at 1731332104999 (+6 ms)Region opened successfully at 1731332105000 (+1 ms) 2024-11-11T13:35:05,007 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731332104882 2024-11-11T13:35:05,018 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T13:35:05,019 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T13:35:05,022 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:05,027 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 37d26ecf6460,36877,1731332101576, state=OPEN 2024-11-11T13:35:05,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T13:35:05,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T13:35:05,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:35:05,069 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:35:05,070 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:05,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T13:35:05,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=37d26ecf6460,36877,1731332101576 in 359 msec 2024-11-11T13:35:05,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T13:35:05,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 612 msec 2024-11-11T13:35:05,085 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T13:35:05,085 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T13:35:05,103 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:05,105 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:05,123 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:05,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34143, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:05,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6570 sec 2024-11-11T13:35:05,148 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731332105148, completionTime=-1 2024-11-11T13:35:05,151 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T13:35:05,151 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T13:35:05,177 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T13:35:05,177 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731332165177 2024-11-11T13:35:05,177 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731332225177 2024-11-11T13:35:05,177 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-11T13:35:05,179 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,35609,1731332100707-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:05,180 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,35609,1731332100707-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:05,180 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,35609,1731332100707-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:05,181 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-37d26ecf6460:35609, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:05,181 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:05,182 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:05,187 DEBUG [master/37d26ecf6460:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T13:35:05,209 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.487sec 2024-11-11T13:35:05,210 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T13:35:05,211 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T13:35:05,212 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T13:35:05,212 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T13:35:05,212 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T13:35:05,213 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,35609,1731332100707-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T13:35:05,214 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,35609,1731332100707-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T13:35:05,222 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T13:35:05,223 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T13:35:05,223 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,35609,1731332100707-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:35:05,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631d4194, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:05,280 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T13:35:05,280 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T13:35:05,283 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:05,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:05,292 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301f7ee4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:05,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:05,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:05,302 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:05,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3557309d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:05,306 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:05,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:05,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=37d26ecf6460,35609,1731332100707 2024-11-11T13:35:05,349 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=3489 2024-11-11T13:35:05,357 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T13:35:05,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 37d26ecf6460,35609,1731332100707 2024-11-11T13:35:05,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b1db67 2024-11-11T13:35:05,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T13:35:05,384 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T13:35:05,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T13:35:05,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:35:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T13:35:05,404 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T13:35:05,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T13:35:05,407 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:05,410 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T13:35:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:35:05,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741835_1011 (size=959) 2024-11-11T13:35:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:35:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:35:05,843 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92 2024-11-11T13:35:05,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741836_1012 (size=53) 2024-11-11T13:35:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:35:06,255 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:35:06,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3b71678035107ffe2dbde3bb5bfff436, disabling compactions & flushes 2024-11-11T13:35:06,256 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:06,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:06,257 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. after waiting 0 ms 2024-11-11T13:35:06,257 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:06,257 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:06,257 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3b71678035107ffe2dbde3bb5bfff436: Waiting for close lock at 1731332106256Disabling compacts and flushes for region at 1731332106256Disabling writes for close at 1731332106257 (+1 ms)Writing region close event to WAL at 1731332106257Closed at 1731332106257 2024-11-11T13:35:06,261 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T13:35:06,268 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731332106261"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731332106261"}]},"ts":"1731332106261"} 2024-11-11T13:35:06,272 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T13:35:06,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T13:35:06,276 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332106274"}]},"ts":"1731332106274"} 2024-11-11T13:35:06,280 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T13:35:06,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3b71678035107ffe2dbde3bb5bfff436, ASSIGN}] 2024-11-11T13:35:06,285 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3b71678035107ffe2dbde3bb5bfff436, ASSIGN 2024-11-11T13:35:06,287 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3b71678035107ffe2dbde3bb5bfff436, ASSIGN; state=OFFLINE, location=37d26ecf6460,36877,1731332101576; forceNewPlan=false, retain=false 2024-11-11T13:35:06,439 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3b71678035107ffe2dbde3bb5bfff436, regionState=OPENING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:06,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3b71678035107ffe2dbde3bb5bfff436, ASSIGN because future has completed 2024-11-11T13:35:06,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:35:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:35:06,606 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:06,607 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:35:06,607 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,607 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:35:06,607 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,608 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,610 INFO [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,613 INFO [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b71678035107ffe2dbde3bb5bfff436 columnFamilyName A 2024-11-11T13:35:06,613 DEBUG [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:06,615 INFO [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] regionserver.HStore(327): Store=3b71678035107ffe2dbde3bb5bfff436/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:06,615 INFO [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,617 INFO [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b71678035107ffe2dbde3bb5bfff436 columnFamilyName B 2024-11-11T13:35:06,617 DEBUG [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:06,618 INFO [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] regionserver.HStore(327): Store=3b71678035107ffe2dbde3bb5bfff436/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:06,618 INFO [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,621 INFO [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b71678035107ffe2dbde3bb5bfff436 columnFamilyName C 2024-11-11T13:35:06,621 DEBUG [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:06,622 INFO [StoreOpener-3b71678035107ffe2dbde3bb5bfff436-1 {}] regionserver.HStore(327): Store=3b71678035107ffe2dbde3bb5bfff436/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:06,622 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,623 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,624 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,626 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,626 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,627 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T13:35:06,629 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,633 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:35:06,634 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3b71678035107ffe2dbde3bb5bfff436; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66088694, jitterRate=-0.015201717615127563}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T13:35:06,634 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:06,635 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3b71678035107ffe2dbde3bb5bfff436: Running coprocessor pre-open hook at 1731332106608Writing region info on filesystem at 1731332106608Initializing all the Stores at 1731332106610 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332106610Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332106610Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332106610Cleaning up temporary data from old regions at 1731332106626 (+16 ms)Running coprocessor post-open hooks at 1731332106634 (+8 ms)Region opened successfully at 1731332106635 (+1 ms) 2024-11-11T13:35:06,637 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., pid=6, masterSystemTime=1731332106599 2024-11-11T13:35:06,640 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:06,640 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:06,641 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3b71678035107ffe2dbde3bb5bfff436, regionState=OPEN, openSeqNum=2, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:06,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:35:06,655 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T13:35:06,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 in 203 msec 2024-11-11T13:35:06,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T13:35:06,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3b71678035107ffe2dbde3bb5bfff436, ASSIGN in 373 msec 2024-11-11T13:35:06,663 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T13:35:06,663 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332106663"}]},"ts":"1731332106663"} 2024-11-11T13:35:06,667 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T13:35:06,669 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T13:35:06,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2750 sec 2024-11-11T13:35:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:35:07,571 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:07,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10296b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:07,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:07,579 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2315f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:07,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:07,584 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35eddcd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,587 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:07,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:07,603 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T13:35:07,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 37d26ecf6460,35609,1731332100707 2024-11-11T13:35:07,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@41fc5204 2024-11-11T13:35:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T13:35:07,615 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T13:35:07,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51c19ecd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:07,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:07,626 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab7eb50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:07,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:07,631 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9c1aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99861ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:07,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:07,637 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703ed473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:07,646 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:07,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67684222, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:07,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:07,652 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef8cf77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:07,657 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af2c0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9bb70a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:07,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:07,666 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:07,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ebcbe76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:07,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:07,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d0905d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2eb13d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:07,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:07,686 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4a2a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:07,691 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738bed33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fa3b57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,699 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:07,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:07,701 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:07,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:07,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50131031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:07,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:07,706 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3981c165, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55e4a390, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,709 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:07,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:07,711 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:07,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35351cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:07,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:07,715 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:07,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@137b074d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c93d1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:07,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:07,720 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:07,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677effe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:07,729 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f2ee53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab8c4ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:07,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:07,733 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:07,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f91c2ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:07,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:07,737 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f4c23c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:07,746 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:07,749 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:07,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:07,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:35:07,760 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:35:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:07,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:07,777 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:07,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:07,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:07,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:35:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:35:07,800 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:07,804 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:07,804 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:07,807 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:07,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:07,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:07,813 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:07,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:07,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:35:07,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:07,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:35:07,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:35:07,832 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:07,836 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:35:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T13:35:07,851 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:07,854 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:07,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T13:35:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:07,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:35:07,980 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:07,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:35:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T13:35:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:08,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T13:35:08,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:08,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/7d94a1098a8c414280f58bedd18487d8 is 50, key is test_row_0/A:col10/1731332107885/Put/seqid=0 2024-11-11T13:35:08,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741837_1013 (size=14341) 2024-11-11T13:35:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T13:35:08,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332168190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332168204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332168217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332168218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332168221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T13:35:08,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332168343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332168342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332168343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332168344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332168350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T13:35:08,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:08,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T13:35:08,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/7d94a1098a8c414280f58bedd18487d8 2024-11-11T13:35:08,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332168560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332168580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332168580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332168581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332168580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T13:35:08,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:08,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/f81b141dcb134e4e95ececba154cc883 is 50, key is test_row_0/B:col10/1731332107885/Put/seqid=0 2024-11-11T13:35:08,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741838_1014 (size=12001) 2024-11-11T13:35:08,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/f81b141dcb134e4e95ececba154cc883 2024-11-11T13:35:08,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/1ac8a326f10b4aeba2d02c5387c3eb48 is 50, key is test_row_0/C:col10/1731332107885/Put/seqid=0 2024-11-11T13:35:08,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T13:35:08,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:08,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741839_1015 (size=12001) 2024-11-11T13:35:08,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/1ac8a326f10b4aeba2d02c5387c3eb48 2024-11-11T13:35:08,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/7d94a1098a8c414280f58bedd18487d8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/7d94a1098a8c414280f58bedd18487d8 2024-11-11T13:35:08,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/7d94a1098a8c414280f58bedd18487d8, entries=200, sequenceid=12, filesize=14.0 K 2024-11-11T13:35:08,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/f81b141dcb134e4e95ececba154cc883 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f81b141dcb134e4e95ececba154cc883 2024-11-11T13:35:08,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f81b141dcb134e4e95ececba154cc883, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T13:35:08,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/1ac8a326f10b4aeba2d02c5387c3eb48 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1ac8a326f10b4aeba2d02c5387c3eb48 2024-11-11T13:35:08,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1ac8a326f10b4aeba2d02c5387c3eb48, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T13:35:08,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 3b71678035107ffe2dbde3bb5bfff436 in 936ms, sequenceid=12, compaction requested=false 2024-11-11T13:35:08,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T13:35:08,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:08,946 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T13:35:08,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:08,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:08,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:08,989 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:08,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=207.98 KB heapSize=545.67 KB 2024-11-11T13:35:08,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332168945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:08,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332168945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f150fd84e0f84371ac94338e407d9d69 is 50, key is test_row_0/A:col10/1731332108205/Put/seqid=0 2024-11-11T13:35:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T13:35:09,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332168991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332168994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332168994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741840_1016 (size=12001) 2024-11-11T13:35:09,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f150fd84e0f84371ac94338e407d9d69 2024-11-11T13:35:09,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T13:35:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:09,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/1ef5950b9f40402081d36b0fd06a5d11 is 50, key is test_row_0/B:col10/1731332108205/Put/seqid=0 2024-11-11T13:35:09,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332169101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332169101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332169145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332169162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741841_1017 (size=12001) 2024-11-11T13:35:09,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332169161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T13:35:09,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:09,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:09,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:09,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332169320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332169321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332169370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332169380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332169390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T13:35:09,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:09,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:09,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:09,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/1ef5950b9f40402081d36b0fd06a5d11 2024-11-11T13:35:09,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T13:35:09,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:09,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:09,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:09,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:09,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/21d7ccff773345359ffe0171cdafe4ad is 50, key is test_row_0/C:col10/1731332108205/Put/seqid=0 2024-11-11T13:35:09,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741842_1018 (size=12001) 2024-11-11T13:35:09,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=69.33 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/21d7ccff773345359ffe0171cdafe4ad 2024-11-11T13:35:09,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f150fd84e0f84371ac94338e407d9d69 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f150fd84e0f84371ac94338e407d9d69 2024-11-11T13:35:09,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332169650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332169651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f150fd84e0f84371ac94338e407d9d69, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T13:35:09,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/1ef5950b9f40402081d36b0fd06a5d11 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1ef5950b9f40402081d36b0fd06a5d11 2024-11-11T13:35:09,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1ef5950b9f40402081d36b0fd06a5d11, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T13:35:09,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/21d7ccff773345359ffe0171cdafe4ad as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/21d7ccff773345359ffe0171cdafe4ad 2024-11-11T13:35:09,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332169693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332169694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332169699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:09,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:09,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:09,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/21d7ccff773345359ffe0171cdafe4ad, entries=150, sequenceid=46, filesize=11.7 K 2024-11-11T13:35:09,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~207.98 KB/212970, heapSize ~545.63 KB/558720, currentSize=0 B/0 for 3b71678035107ffe2dbde3bb5bfff436 in 717ms, sequenceid=46, compaction requested=false 2024-11-11T13:35:09,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:09,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T13:35:09,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:09,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:09,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:09,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T13:35:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T13:35:09,753 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T13:35:09,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T13:35:09,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8920 sec 2024-11-11T13:35:09,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9340 sec 2024-11-11T13:35:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T13:35:10,033 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:10,033 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T13:35:10,037 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:10,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:10,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T13:35:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T13:35:10,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T13:35:10,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:10,199 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-11T13:35:10,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/366d417dec3a42bdad8fbfc6ee8ec80a is 50, key is test_row_0/A:col10/1731332110188/Put/seqid=0 2024-11-11T13:35:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:10,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:10,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741843_1019 (size=12001) 2024-11-11T13:35:10,256 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/366d417dec3a42bdad8fbfc6ee8ec80a 2024-11-11T13:35:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/23c560098cb746e2ad32b1ea302af89a is 50, key is test_row_0/B:col10/1731332110188/Put/seqid=0 2024-11-11T13:35:10,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741844_1020 (size=12001) 2024-11-11T13:35:10,334 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/23c560098cb746e2ad32b1ea302af89a 2024-11-11T13:35:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T13:35:10,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332170355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/c4dc4263c250431c8e19032b187a3edd is 50, key is test_row_0/C:col10/1731332110188/Put/seqid=0 2024-11-11T13:35:10,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332170360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332170360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332170363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332170363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741845_1021 (size=12001) 2024-11-11T13:35:10,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332170470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332170480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,482 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T13:35:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332170481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332170481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332170489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T13:35:10,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332170680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332170689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332170690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332170691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332170699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:10,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:10,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:10,801 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/c4dc4263c250431c8e19032b187a3edd 2024-11-11T13:35:10,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/366d417dec3a42bdad8fbfc6ee8ec80a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/366d417dec3a42bdad8fbfc6ee8ec80a 2024-11-11T13:35:10,851 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/366d417dec3a42bdad8fbfc6ee8ec80a, entries=150, sequenceid=55, filesize=11.7 K 2024-11-11T13:35:10,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/23c560098cb746e2ad32b1ea302af89a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/23c560098cb746e2ad32b1ea302af89a 2024-11-11T13:35:10,884 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/23c560098cb746e2ad32b1ea302af89a, entries=150, sequenceid=55, filesize=11.7 K 2024-11-11T13:35:10,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/c4dc4263c250431c8e19032b187a3edd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c4dc4263c250431c8e19032b187a3edd 2024-11-11T13:35:10,907 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c4dc4263c250431c8e19032b187a3edd, entries=150, sequenceid=55, filesize=11.7 K 2024-11-11T13:35:10,912 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 3b71678035107ffe2dbde3bb5bfff436 in 713ms, sequenceid=55, compaction requested=true 2024-11-11T13:35:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T13:35:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T13:35:10,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T13:35:10,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 881 msec 2024-11-11T13:35:10,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 907 msec 2024-11-11T13:35:11,000 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:11,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-11T13:35:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:11,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/590b3ee34e694feea661d748bf37ce8a is 50, key is test_row_0/A:col10/1731332110997/Put/seqid=0 2024-11-11T13:35:11,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332171011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332171012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332171015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332171022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332171023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741846_1022 (size=12001) 2024-11-11T13:35:11,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/590b3ee34e694feea661d748bf37ce8a 2024-11-11T13:35:11,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/c07b03fbbe5b4e09ac08e530db4c326d is 50, key is test_row_0/B:col10/1731332110997/Put/seqid=0 2024-11-11T13:35:11,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332171130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332171131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332171140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332171140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741847_1023 (size=12001) 2024-11-11T13:35:11,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/c07b03fbbe5b4e09ac08e530db4c326d 2024-11-11T13:35:11,161 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T13:35:11,162 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T13:35:11,164 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T13:35:11,165 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T13:35:11,165 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T13:35:11,165 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T13:35:11,167 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T13:35:11,167 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T13:35:11,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/00a067dfdf2d480189b777c40946325e is 50, key is test_row_0/C:col10/1731332110997/Put/seqid=0 2024-11-11T13:35:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T13:35:11,190 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:11,191 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T13:35:11,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T13:35:11,195 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:11,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:11,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741848_1024 (size=12001) 2024-11-11T13:35:11,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/00a067dfdf2d480189b777c40946325e 2024-11-11T13:35:11,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/590b3ee34e694feea661d748bf37ce8a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/590b3ee34e694feea661d748bf37ce8a 2024-11-11T13:35:11,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T13:35:11,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/590b3ee34e694feea661d748bf37ce8a, entries=150, sequenceid=85, filesize=11.7 K 2024-11-11T13:35:11,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/c07b03fbbe5b4e09ac08e530db4c326d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c07b03fbbe5b4e09ac08e530db4c326d 2024-11-11T13:35:11,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T13:35:11,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:11,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:11,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:11,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:11,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332171340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332171350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332171350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332171351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c07b03fbbe5b4e09ac08e530db4c326d, entries=150, sequenceid=85, filesize=11.7 K 2024-11-11T13:35:11,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/00a067dfdf2d480189b777c40946325e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/00a067dfdf2d480189b777c40946325e 2024-11-11T13:35:11,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/00a067dfdf2d480189b777c40946325e, entries=150, sequenceid=85, filesize=11.7 K 2024-11-11T13:35:11,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for 3b71678035107ffe2dbde3bb5bfff436 in 435ms, sequenceid=85, compaction requested=true 2024-11-11T13:35:11,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:11,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:11,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:11,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:11,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:11,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:11,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:11,445 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:11,451 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:11,465 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:11,467 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/B is initiating minor compaction (all files) 2024-11-11T13:35:11,468 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/B in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:11,468 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f81b141dcb134e4e95ececba154cc883, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1ef5950b9f40402081d36b0fd06a5d11, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/23c560098cb746e2ad32b1ea302af89a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c07b03fbbe5b4e09ac08e530db4c326d] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=46.9 K 2024-11-11T13:35:11,470 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:11,470 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/A is initiating minor compaction (all files) 2024-11-11T13:35:11,471 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/A in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:11,471 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/7d94a1098a8c414280f58bedd18487d8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f150fd84e0f84371ac94338e407d9d69, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/366d417dec3a42bdad8fbfc6ee8ec80a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/590b3ee34e694feea661d748bf37ce8a] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=49.2 K 2024-11-11T13:35:11,478 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d94a1098a8c414280f58bedd18487d8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731332107885 2024-11-11T13:35:11,479 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f81b141dcb134e4e95ececba154cc883, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731332107885 2024-11-11T13:35:11,494 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f150fd84e0f84371ac94338e407d9d69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731332108205 2024-11-11T13:35:11,495 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ef5950b9f40402081d36b0fd06a5d11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731332108205 2024-11-11T13:35:11,497 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 366d417dec3a42bdad8fbfc6ee8ec80a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731332110182 2024-11-11T13:35:11,498 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 23c560098cb746e2ad32b1ea302af89a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731332110182 2024-11-11T13:35:11,501 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c07b03fbbe5b4e09ac08e530db4c326d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731332110328 2024-11-11T13:35:11,502 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 590b3ee34e694feea661d748bf37ce8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731332110328 2024-11-11T13:35:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T13:35:11,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T13:35:11,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:11,513 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-11T13:35:11,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/6c075679de414f09a829276642146e96 is 50, key is test_row_0/A:col10/1731332111017/Put/seqid=0 2024-11-11T13:35:11,555 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#B#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:11,556 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/4059bb934ec94f87b8ef12fed7c9d270 is 50, key is test_row_0/B:col10/1731332110997/Put/seqid=0 2024-11-11T13:35:11,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741849_1025 (size=12001) 2024-11-11T13:35:11,595 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#A#compaction#14 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:11,596 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/9f7f444dbe7b4227942c3413c72f8f0f is 50, key is test_row_0/A:col10/1731332110997/Put/seqid=0 2024-11-11T13:35:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:11,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:11,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741850_1026 (size=12139) 2024-11-11T13:35:11,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741851_1027 (size=12139) 2024-11-11T13:35:11,678 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/4059bb934ec94f87b8ef12fed7c9d270 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/4059bb934ec94f87b8ef12fed7c9d270 2024-11-11T13:35:11,687 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/9f7f444dbe7b4227942c3413c72f8f0f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9f7f444dbe7b4227942c3413c72f8f0f 2024-11-11T13:35:11,725 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/A of 3b71678035107ffe2dbde3bb5bfff436 into 9f7f444dbe7b4227942c3413c72f8f0f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:11,725 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:11,727 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/B of 3b71678035107ffe2dbde3bb5bfff436 into 4059bb934ec94f87b8ef12fed7c9d270(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:11,727 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:11,730 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/B, priority=12, startTime=1731332111441; duration=0sec 2024-11-11T13:35:11,730 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/A, priority=12, startTime=1731332111438; duration=0sec 2024-11-11T13:35:11,731 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:11,731 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:11,731 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:11,731 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:11,732 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:11,736 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:11,736 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/C is initiating minor compaction (all files) 2024-11-11T13:35:11,736 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/C in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:11,737 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1ac8a326f10b4aeba2d02c5387c3eb48, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/21d7ccff773345359ffe0171cdafe4ad, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c4dc4263c250431c8e19032b187a3edd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/00a067dfdf2d480189b777c40946325e] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=46.9 K 2024-11-11T13:35:11,740 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac8a326f10b4aeba2d02c5387c3eb48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731332107885 2024-11-11T13:35:11,741 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d7ccff773345359ffe0171cdafe4ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731332108205 2024-11-11T13:35:11,744 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c4dc4263c250431c8e19032b187a3edd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731332110182 2024-11-11T13:35:11,747 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a067dfdf2d480189b777c40946325e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731332110328 2024-11-11T13:35:11,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332171782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332171785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332171789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,798 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#C#compaction#15 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:11,799 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/ad377a97a035475e93c43cc0fbaf6caa is 50, key is test_row_0/C:col10/1731332110997/Put/seqid=0 2024-11-11T13:35:11,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332171791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332171791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T13:35:11,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741852_1028 (size=12139) 2024-11-11T13:35:11,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332171912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332171912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332171912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332171912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332171913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:11,986 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/6c075679de414f09a829276642146e96 2024-11-11T13:35:12,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/1bae6447da714f40beda8c786853ed2a is 50, key is test_row_0/B:col10/1731332111017/Put/seqid=0 2024-11-11T13:35:12,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741853_1029 (size=12001) 2024-11-11T13:35:12,077 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/1bae6447da714f40beda8c786853ed2a 2024-11-11T13:35:12,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/adfbc1e598214aa2b295d4d26aa70b2d is 50, key is test_row_0/C:col10/1731332111017/Put/seqid=0 2024-11-11T13:35:12,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332172141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332172141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332172141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332172142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332172141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741854_1030 (size=12001) 2024-11-11T13:35:12,149 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/adfbc1e598214aa2b295d4d26aa70b2d 2024-11-11T13:35:12,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/6c075679de414f09a829276642146e96 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6c075679de414f09a829276642146e96 2024-11-11T13:35:12,180 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6c075679de414f09a829276642146e96, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T13:35:12,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/1bae6447da714f40beda8c786853ed2a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1bae6447da714f40beda8c786853ed2a 2024-11-11T13:35:12,227 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1bae6447da714f40beda8c786853ed2a, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T13:35:12,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/adfbc1e598214aa2b295d4d26aa70b2d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/adfbc1e598214aa2b295d4d26aa70b2d 2024-11-11T13:35:12,270 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/adfbc1e598214aa2b295d4d26aa70b2d, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T13:35:12,277 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/ad377a97a035475e93c43cc0fbaf6caa as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/ad377a97a035475e93c43cc0fbaf6caa 2024-11-11T13:35:12,280 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 3b71678035107ffe2dbde3bb5bfff436 in 767ms, sequenceid=92, compaction requested=false 2024-11-11T13:35:12,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:12,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:12,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T13:35:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T13:35:12,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T13:35:12,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.1000 sec 2024-11-11T13:35:12,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0870 sec 2024-11-11T13:35:12,328 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/C of 3b71678035107ffe2dbde3bb5bfff436 into ad377a97a035475e93c43cc0fbaf6caa(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:12,329 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:12,329 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/C, priority=12, startTime=1731332111442; duration=0sec 2024-11-11T13:35:12,329 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T13:35:12,329 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:12,330 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:12,331 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T13:35:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T13:35:12,359 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:12,364 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:12,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T13:35:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:12,479 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:12,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-11T13:35:12,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332172480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332172489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332172489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/74e0aab0441d407caad081cf6df795db is 50, key is test_row_0/A:col10/1731332112458/Put/seqid=0 2024-11-11T13:35:12,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332172511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332172509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T13:35:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:12,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:12,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741855_1031 (size=12001) 2024-11-11T13:35:12,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/74e0aab0441d407caad081cf6df795db 2024-11-11T13:35:12,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/18342b3039794313bc568d208cc042fa is 50, key is test_row_0/B:col10/1731332112458/Put/seqid=0 2024-11-11T13:35:12,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741856_1032 (size=12001) 2024-11-11T13:35:12,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332172619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332172630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332172630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332172630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332172639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T13:35:12,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T13:35:12,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:12,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:12,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:12,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:12,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:12,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332172829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332172839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332172840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332172840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T13:35:12,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332172852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:12,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:12,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:12,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T13:35:12,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/18342b3039794313bc568d208cc042fa 2024-11-11T13:35:13,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T13:35:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/6ca9b23a19a743c2ad158e5640aa2356 is 50, key is test_row_0/C:col10/1731332112458/Put/seqid=0 2024-11-11T13:35:13,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741857_1033 (size=12001) 2024-11-11T13:35:13,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332173144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:13,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332173156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:13,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:13,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332173156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:13,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:13,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T13:35:13,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332173160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:13,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:13,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:13,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332173160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:13,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T13:35:13,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:13,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T13:35:13,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T13:35:13,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:13,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:13,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/6ca9b23a19a743c2ad158e5640aa2356 2024-11-11T13:35:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/74e0aab0441d407caad081cf6df795db as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/74e0aab0441d407caad081cf6df795db 2024-11-11T13:35:13,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/74e0aab0441d407caad081cf6df795db, entries=150, sequenceid=127, filesize=11.7 K 2024-11-11T13:35:13,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/18342b3039794313bc568d208cc042fa as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/18342b3039794313bc568d208cc042fa 2024-11-11T13:35:13,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/18342b3039794313bc568d208cc042fa, entries=150, sequenceid=127, filesize=11.7 K 2024-11-11T13:35:13,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/6ca9b23a19a743c2ad158e5640aa2356 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6ca9b23a19a743c2ad158e5640aa2356 2024-11-11T13:35:13,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6ca9b23a19a743c2ad158e5640aa2356, entries=150, sequenceid=127, filesize=11.7 K 2024-11-11T13:35:13,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for 3b71678035107ffe2dbde3bb5bfff436 in 1122ms, sequenceid=127, compaction requested=true 2024-11-11T13:35:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:13,602 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:13,602 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:13,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:13,609 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:13,609 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/A is initiating minor compaction (all files) 2024-11-11T13:35:13,609 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/A in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,609 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9f7f444dbe7b4227942c3413c72f8f0f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6c075679de414f09a829276642146e96, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/74e0aab0441d407caad081cf6df795db] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=35.3 K 2024-11-11T13:35:13,610 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f7f444dbe7b4227942c3413c72f8f0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731332110328 2024-11-11T13:35:13,611 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c075679de414f09a829276642146e96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731332111006 2024-11-11T13:35:13,612 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74e0aab0441d407caad081cf6df795db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731332111787 2024-11-11T13:35:13,613 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:13,613 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/B is initiating minor compaction (all files) 2024-11-11T13:35:13,613 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/B in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,614 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/4059bb934ec94f87b8ef12fed7c9d270, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1bae6447da714f40beda8c786853ed2a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/18342b3039794313bc568d208cc042fa] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=35.3 K 2024-11-11T13:35:13,615 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 4059bb934ec94f87b8ef12fed7c9d270, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731332110328 2024-11-11T13:35:13,616 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bae6447da714f40beda8c786853ed2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731332111006 2024-11-11T13:35:13,618 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 18342b3039794313bc568d208cc042fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731332111787 2024-11-11T13:35:13,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T13:35:13,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,645 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-11T13:35:13,667 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#A#compaction#21 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:13,668 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/4dd0490712a640f1a61cc3717da1d005 is 50, key is test_row_0/A:col10/1731332112458/Put/seqid=0 2024-11-11T13:35:13,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/9ec3341bcf984171aed0424bbb2fb8bc is 50, key is test_row_2/A:col10/1731332112495/Put/seqid=0 2024-11-11T13:35:13,673 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#B#compaction#23 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:13,674 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/5cf5c690da274370add1892444a9ae4f is 50, key is test_row_0/B:col10/1731332112458/Put/seqid=0 2024-11-11T13:35:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:13,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:13,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741860_1036 (size=12241) 2024-11-11T13:35:13,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741858_1034 (size=7365) 2024-11-11T13:35:13,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741859_1035 (size=12241) 2024-11-11T13:35:13,788 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/9ec3341bcf984171aed0424bbb2fb8bc 2024-11-11T13:35:13,788 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/5cf5c690da274370add1892444a9ae4f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/5cf5c690da274370add1892444a9ae4f 2024-11-11T13:35:13,807 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/4dd0490712a640f1a61cc3717da1d005 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/4dd0490712a640f1a61cc3717da1d005 2024-11-11T13:35:13,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/0937dc0df14947c696bbe21f458892fd is 50, key is test_row_2/B:col10/1731332112495/Put/seqid=0 2024-11-11T13:35:13,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741861_1037 (size=7365) 2024-11-11T13:35:13,832 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/0937dc0df14947c696bbe21f458892fd 2024-11-11T13:35:13,844 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/B of 3b71678035107ffe2dbde3bb5bfff436 into 5cf5c690da274370add1892444a9ae4f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:13,845 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:13,845 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/B, priority=13, startTime=1731332113602; duration=0sec 2024-11-11T13:35:13,845 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:13,845 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:13,845 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:13,847 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/A of 3b71678035107ffe2dbde3bb5bfff436 into 4dd0490712a640f1a61cc3717da1d005(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:13,847 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:13,847 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/A, priority=13, startTime=1731332113602; duration=0sec 2024-11-11T13:35:13,848 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:13,848 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:13,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/9b4ca4270873498b8f7369a5dec05a3f is 50, key is test_row_2/C:col10/1731332112495/Put/seqid=0 2024-11-11T13:35:13,855 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:13,855 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/C is initiating minor compaction (all files) 2024-11-11T13:35:13,855 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/C in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,856 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/ad377a97a035475e93c43cc0fbaf6caa, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/adfbc1e598214aa2b295d4d26aa70b2d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6ca9b23a19a743c2ad158e5640aa2356] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=35.3 K 2024-11-11T13:35:13,858 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting ad377a97a035475e93c43cc0fbaf6caa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731332110328 2024-11-11T13:35:13,859 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting adfbc1e598214aa2b295d4d26aa70b2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731332111006 2024-11-11T13:35:13,859 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ca9b23a19a743c2ad158e5640aa2356, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731332111787 2024-11-11T13:35:13,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741862_1038 (size=7365) 2024-11-11T13:35:13,877 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/9b4ca4270873498b8f7369a5dec05a3f 2024-11-11T13:35:13,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332173875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:13,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332173879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:13,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:13,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332173876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:13,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332173884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:13,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332173884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:13,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/9ec3341bcf984171aed0424bbb2fb8bc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9ec3341bcf984171aed0424bbb2fb8bc 2024-11-11T13:35:13,899 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#C#compaction#26 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:13,905 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/6c162a50c4524782bc9840b19cb7de25 is 50, key is test_row_0/C:col10/1731332112458/Put/seqid=0 2024-11-11T13:35:13,914 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9ec3341bcf984171aed0424bbb2fb8bc, entries=50, sequenceid=132, filesize=7.2 K 2024-11-11T13:35:13,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741863_1039 (size=12241) 2024-11-11T13:35:13,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/0937dc0df14947c696bbe21f458892fd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0937dc0df14947c696bbe21f458892fd 2024-11-11T13:35:13,932 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0937dc0df14947c696bbe21f458892fd, entries=50, sequenceid=132, filesize=7.2 K 2024-11-11T13:35:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/9b4ca4270873498b8f7369a5dec05a3f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/9b4ca4270873498b8f7369a5dec05a3f 2024-11-11T13:35:13,949 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/9b4ca4270873498b8f7369a5dec05a3f, entries=50, sequenceid=132, filesize=7.2 K 2024-11-11T13:35:13,951 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=187.85 KB/192360 for 3b71678035107ffe2dbde3bb5bfff436 in 307ms, sequenceid=132, compaction requested=false 2024-11-11T13:35:13,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:13,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:13,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T13:35:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T13:35:13,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T13:35:13,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5900 sec 2024-11-11T13:35:13,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6290 sec 2024-11-11T13:35:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:13,994 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:13,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-11T13:35:13,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332173989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:13,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:13,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332173996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332173996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332174001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/2069050e8e33433ab201752012aa7304 is 50, key is test_row_0/A:col10/1731332113881/Put/seqid=0 2024-11-11T13:35:14,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332174003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741864_1040 (size=16931) 2024-11-11T13:35:14,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/2069050e8e33433ab201752012aa7304 2024-11-11T13:35:14,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/6d89665b683e4fc58b2a4798356ab7f2 is 50, key is test_row_0/B:col10/1731332113881/Put/seqid=0 2024-11-11T13:35:14,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741865_1041 (size=12151) 2024-11-11T13:35:14,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/6d89665b683e4fc58b2a4798356ab7f2 2024-11-11T13:35:14,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/06baa5541c7643f4a793980d1464ae36 is 50, key is test_row_0/C:col10/1731332113881/Put/seqid=0 2024-11-11T13:35:14,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332174112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332174112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741866_1042 (size=12151) 2024-11-11T13:35:14,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/06baa5541c7643f4a793980d1464ae36 2024-11-11T13:35:14,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/2069050e8e33433ab201752012aa7304 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/2069050e8e33433ab201752012aa7304 2024-11-11T13:35:14,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/2069050e8e33433ab201752012aa7304, entries=250, sequenceid=167, filesize=16.5 K 2024-11-11T13:35:14,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/6d89665b683e4fc58b2a4798356ab7f2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/6d89665b683e4fc58b2a4798356ab7f2 2024-11-11T13:35:14,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/6d89665b683e4fc58b2a4798356ab7f2, entries=150, sequenceid=167, filesize=11.9 K 2024-11-11T13:35:14,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/06baa5541c7643f4a793980d1464ae36 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/06baa5541c7643f4a793980d1464ae36 2024-11-11T13:35:14,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/06baa5541c7643f4a793980d1464ae36, entries=150, sequenceid=167, filesize=11.9 K 2024-11-11T13:35:14,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 3b71678035107ffe2dbde3bb5bfff436 in 198ms, sequenceid=167, compaction requested=true 2024-11-11T13:35:14,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:14,193 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:14,194 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:14,195 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/A is initiating minor compaction (all files) 2024-11-11T13:35:14,195 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/A in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:14,195 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/4dd0490712a640f1a61cc3717da1d005, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9ec3341bcf984171aed0424bbb2fb8bc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/2069050e8e33433ab201752012aa7304] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=35.7 K 2024-11-11T13:35:14,196 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dd0490712a640f1a61cc3717da1d005, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731332111787 2024-11-11T13:35:14,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:14,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:14,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:14,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:14,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:14,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:35:14,201 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ec3341bcf984171aed0424bbb2fb8bc, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731332112495 2024-11-11T13:35:14,201 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2069050e8e33433ab201752012aa7304, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731332113861 2024-11-11T13:35:14,221 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#A#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:14,222 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/06890239400c4b508cded0becde4bdb3 is 50, key is test_row_0/A:col10/1731332113881/Put/seqid=0 2024-11-11T13:35:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:14,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741867_1043 (size=12493) 2024-11-11T13:35:14,232 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:14,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:35:14,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/9149189ddbbd4634818fe72e4f6293e4 is 50, key is test_row_0/A:col10/1731332114213/Put/seqid=0 2024-11-11T13:35:14,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741868_1044 (size=12151) 2024-11-11T13:35:14,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332174310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332174310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332174312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332174320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332174322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,332 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/6c162a50c4524782bc9840b19cb7de25 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6c162a50c4524782bc9840b19cb7de25 2024-11-11T13:35:14,348 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/C of 3b71678035107ffe2dbde3bb5bfff436 into 6c162a50c4524782bc9840b19cb7de25(size=12.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:14,348 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:14,348 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/C, priority=13, startTime=1731332113602; duration=0sec 2024-11-11T13:35:14,348 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:35:14,349 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:14,349 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:14,349 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:14,352 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31757 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:14,352 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/B is initiating minor compaction (all files) 2024-11-11T13:35:14,352 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/B in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:14,353 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/5cf5c690da274370add1892444a9ae4f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0937dc0df14947c696bbe21f458892fd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/6d89665b683e4fc58b2a4798356ab7f2] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=31.0 K 2024-11-11T13:35:14,354 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf5c690da274370add1892444a9ae4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731332111787 2024-11-11T13:35:14,355 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 0937dc0df14947c696bbe21f458892fd, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731332112495 2024-11-11T13:35:14,355 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d89665b683e4fc58b2a4798356ab7f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731332113881 2024-11-11T13:35:14,389 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#B#compaction#32 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:14,390 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/0d205f47a16344b794b48d26a2b2e7be is 50, key is test_row_0/B:col10/1731332113881/Put/seqid=0 2024-11-11T13:35:14,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741869_1045 (size=12493) 2024-11-11T13:35:14,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332174420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332174429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332174429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,446 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/0d205f47a16344b794b48d26a2b2e7be as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0d205f47a16344b794b48d26a2b2e7be 2024-11-11T13:35:14,467 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/B of 3b71678035107ffe2dbde3bb5bfff436 into 0d205f47a16344b794b48d26a2b2e7be(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:14,467 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:14,467 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/B, priority=13, startTime=1731332114196; duration=0sec 2024-11-11T13:35:14,467 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:14,467 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:14,467 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:14,478 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31757 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:14,478 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/C is initiating minor compaction (all files) 2024-11-11T13:35:14,478 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/C in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:14,478 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6c162a50c4524782bc9840b19cb7de25, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/9b4ca4270873498b8f7369a5dec05a3f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/06baa5541c7643f4a793980d1464ae36] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=31.0 K 2024-11-11T13:35:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T13:35:14,480 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:14,482 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:14,482 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c162a50c4524782bc9840b19cb7de25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731332111787 2024-11-11T13:35:14,483 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b4ca4270873498b8f7369a5dec05a3f, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731332112495 2024-11-11T13:35:14,484 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 06baa5541c7643f4a793980d1464ae36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731332113881 2024-11-11T13:35:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T13:35:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T13:35:14,486 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:14,495 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:14,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:14,559 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#C#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:14,559 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/b64a5f83d04e43699245177b112484cf is 50, key is test_row_0/C:col10/1731332113881/Put/seqid=0 2024-11-11T13:35:14,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741870_1046 (size=12493) 2024-11-11T13:35:14,586 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/b64a5f83d04e43699245177b112484cf as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/b64a5f83d04e43699245177b112484cf 2024-11-11T13:35:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T13:35:14,600 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/C of 3b71678035107ffe2dbde3bb5bfff436 into b64a5f83d04e43699245177b112484cf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:14,600 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:14,601 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/C, priority=13, startTime=1731332114197; duration=0sec 2024-11-11T13:35:14,601 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:14,601 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:14,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332174629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332174629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,642 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/06890239400c4b508cded0becde4bdb3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/06890239400c4b508cded0becde4bdb3 2024-11-11T13:35:14,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332174640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T13:35:14,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332174649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:14,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:14,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332174650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:14,657 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/A of 3b71678035107ffe2dbde3bb5bfff436 into 06890239400c4b508cded0becde4bdb3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:14,657 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:14,657 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/A, priority=13, startTime=1731332114192; duration=0sec 2024-11-11T13:35:14,657 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:14,657 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:14,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/9149189ddbbd4634818fe72e4f6293e4 2024-11-11T13:35:14,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/87624ab6620242c0a9a204034f1a0abe is 50, key is test_row_0/B:col10/1731332114213/Put/seqid=0 2024-11-11T13:35:14,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741871_1047 (size=12151) 2024-11-11T13:35:14,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/87624ab6620242c0a9a204034f1a0abe 2024-11-11T13:35:14,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/14246d5ef6414357832527a65870c2e3 is 50, key is test_row_0/C:col10/1731332114213/Put/seqid=0 2024-11-11T13:35:14,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741872_1048 (size=12151) 2024-11-11T13:35:14,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/14246d5ef6414357832527a65870c2e3 2024-11-11T13:35:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T13:35:14,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/9149189ddbbd4634818fe72e4f6293e4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9149189ddbbd4634818fe72e4f6293e4 2024-11-11T13:35:14,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T13:35:14,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:14,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:14,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:14,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:14,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:14,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9149189ddbbd4634818fe72e4f6293e4, entries=150, sequenceid=178, filesize=11.9 K 2024-11-11T13:35:14,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/87624ab6620242c0a9a204034f1a0abe as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/87624ab6620242c0a9a204034f1a0abe 2024-11-11T13:35:14,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/87624ab6620242c0a9a204034f1a0abe, entries=150, sequenceid=178, filesize=11.9 K 2024-11-11T13:35:14,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/14246d5ef6414357832527a65870c2e3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/14246d5ef6414357832527a65870c2e3 2024-11-11T13:35:14,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/14246d5ef6414357832527a65870c2e3, entries=150, sequenceid=178, filesize=11.9 K 2024-11-11T13:35:14,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3b71678035107ffe2dbde3bb5bfff436 in 632ms, sequenceid=178, compaction requested=false 2024-11-11T13:35:14,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:14,943 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:14,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T13:35:14,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/176c26a3cc354d058292ff93830f7d87 is 50, key is test_row_0/A:col10/1731332114301/Put/seqid=0 2024-11-11T13:35:14,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T13:35:14,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:14,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:14,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:14,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:14,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:14,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741873_1049 (size=12151) 2024-11-11T13:35:14,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/176c26a3cc354d058292ff93830f7d87 2024-11-11T13:35:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332174986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332174989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332174990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/3fa3487f9b3b4f288a03a64532a2b975 is 50, key is test_row_0/B:col10/1731332114301/Put/seqid=0 2024-11-11T13:35:15,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741874_1050 (size=12151) 2024-11-11T13:35:15,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332175104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:15,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T13:35:15,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332175104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332175104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T13:35:15,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:15,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332175141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:15,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332175150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T13:35:15,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:15,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332175331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332175331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332175330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/3fa3487f9b3b4f288a03a64532a2b975 2024-11-11T13:35:15,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T13:35:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/5189fb7cc44f4aeca211004e7c87ab5f is 50, key is test_row_0/C:col10/1731332114301/Put/seqid=0 2024-11-11T13:35:15,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741875_1051 (size=12151) 2024-11-11T13:35:15,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/5189fb7cc44f4aeca211004e7c87ab5f 2024-11-11T13:35:15,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/176c26a3cc354d058292ff93830f7d87 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/176c26a3cc354d058292ff93830f7d87 2024-11-11T13:35:15,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/176c26a3cc354d058292ff93830f7d87, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T13:35:15,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/3fa3487f9b3b4f288a03a64532a2b975 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3fa3487f9b3b4f288a03a64532a2b975 2024-11-11T13:35:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T13:35:15,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T13:35:15,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:15,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:15,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332175639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:15,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332175639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3fa3487f9b3b4f288a03a64532a2b975, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T13:35:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332175640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:15,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/5189fb7cc44f4aeca211004e7c87ab5f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/5189fb7cc44f4aeca211004e7c87ab5f 2024-11-11T13:35:15,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/5189fb7cc44f4aeca211004e7c87ab5f, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T13:35:15,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for 3b71678035107ffe2dbde3bb5bfff436 in 728ms, sequenceid=208, compaction requested=true 2024-11-11T13:35:15,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:15,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:15,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:15,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:15,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:35:15,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:15,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T13:35:15,673 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:15,674 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:15,674 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/A is initiating minor compaction (all files) 2024-11-11T13:35:15,674 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/A in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,674 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/06890239400c4b508cded0becde4bdb3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9149189ddbbd4634818fe72e4f6293e4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/176c26a3cc354d058292ff93830f7d87] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=35.9 K 2024-11-11T13:35:15,675 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06890239400c4b508cded0becde4bdb3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731332113881 2024-11-11T13:35:15,675 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:15,676 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9149189ddbbd4634818fe72e4f6293e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731332114213 2024-11-11T13:35:15,676 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 176c26a3cc354d058292ff93830f7d87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731332114301 2024-11-11T13:35:15,684 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:15,684 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/C is initiating minor compaction (all files) 2024-11-11T13:35:15,685 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/C in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,685 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/b64a5f83d04e43699245177b112484cf, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/14246d5ef6414357832527a65870c2e3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/5189fb7cc44f4aeca211004e7c87ab5f] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=35.9 K 2024-11-11T13:35:15,686 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting b64a5f83d04e43699245177b112484cf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731332113881 2024-11-11T13:35:15,687 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 14246d5ef6414357832527a65870c2e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731332114213 2024-11-11T13:35:15,691 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 5189fb7cc44f4aeca211004e7c87ab5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731332114301 2024-11-11T13:35:15,706 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#A#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:15,707 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/e205e7716181493f9a8c5b4a6d78144a is 50, key is test_row_0/A:col10/1731332114301/Put/seqid=0 2024-11-11T13:35:15,714 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#C#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:15,714 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/dd6513afb476422297a9bf34664059e6 is 50, key is test_row_0/C:col10/1731332114301/Put/seqid=0 2024-11-11T13:35:15,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741877_1053 (size=12595) 2024-11-11T13:35:15,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741876_1052 (size=12595) 2024-11-11T13:35:15,740 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/dd6513afb476422297a9bf34664059e6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/dd6513afb476422297a9bf34664059e6 2024-11-11T13:35:15,748 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/e205e7716181493f9a8c5b4a6d78144a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/e205e7716181493f9a8c5b4a6d78144a 2024-11-11T13:35:15,776 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/C of 3b71678035107ffe2dbde3bb5bfff436 into dd6513afb476422297a9bf34664059e6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:15,776 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:15,776 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/C, priority=13, startTime=1731332115672; duration=0sec 2024-11-11T13:35:15,776 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:15,776 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:15,776 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:15,779 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/A of 3b71678035107ffe2dbde3bb5bfff436 into e205e7716181493f9a8c5b4a6d78144a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:15,779 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:15,780 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/A, priority=13, startTime=1731332115671; duration=0sec 2024-11-11T13:35:15,780 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:15,780 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:15,783 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:15,783 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/B is initiating minor compaction (all files) 2024-11-11T13:35:15,784 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/B in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,784 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0d205f47a16344b794b48d26a2b2e7be, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/87624ab6620242c0a9a204034f1a0abe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3fa3487f9b3b4f288a03a64532a2b975] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=35.9 K 2024-11-11T13:35:15,785 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d205f47a16344b794b48d26a2b2e7be, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731332113881 2024-11-11T13:35:15,786 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 87624ab6620242c0a9a204034f1a0abe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731332114213 2024-11-11T13:35:15,787 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fa3487f9b3b4f288a03a64532a2b975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731332114301 2024-11-11T13:35:15,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T13:35:15,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:15,796 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T13:35:15,813 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#B#compaction#41 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:15,814 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/cb4d57be9f7d4394a608f0ddf1ea0af6 is 50, key is test_row_0/B:col10/1731332114301/Put/seqid=0 2024-11-11T13:35:15,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f3fdef481c3e4ec483eaba9860806e17 is 50, key is test_row_0/A:col10/1731332114984/Put/seqid=0 2024-11-11T13:35:15,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741879_1055 (size=12151) 2024-11-11T13:35:15,896 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f3fdef481c3e4ec483eaba9860806e17 2024-11-11T13:35:15,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741878_1054 (size=12595) 2024-11-11T13:35:15,928 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/cb4d57be9f7d4394a608f0ddf1ea0af6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/cb4d57be9f7d4394a608f0ddf1ea0af6 2024-11-11T13:35:15,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/3a656b7bdc864291a3833ed3db117ae6 is 50, key is test_row_0/B:col10/1731332114984/Put/seqid=0 2024-11-11T13:35:15,949 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/B of 3b71678035107ffe2dbde3bb5bfff436 into cb4d57be9f7d4394a608f0ddf1ea0af6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:15,949 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:15,949 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/B, priority=13, startTime=1731332115671; duration=0sec 2024-11-11T13:35:15,949 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:15,949 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:15,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741880_1056 (size=12151) 2024-11-11T13:35:15,982 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/3a656b7bdc864291a3833ed3db117ae6 2024-11-11T13:35:16,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/4406a0b68e8d444c95612ff0117c926b is 50, key is test_row_0/C:col10/1731332114984/Put/seqid=0 2024-11-11T13:35:16,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741881_1057 (size=12151) 2024-11-11T13:35:16,063 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/4406a0b68e8d444c95612ff0117c926b 2024-11-11T13:35:16,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f3fdef481c3e4ec483eaba9860806e17 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f3fdef481c3e4ec483eaba9860806e17 2024-11-11T13:35:16,088 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f3fdef481c3e4ec483eaba9860806e17, entries=150, sequenceid=222, filesize=11.9 K 2024-11-11T13:35:16,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/3a656b7bdc864291a3833ed3db117ae6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3a656b7bdc864291a3833ed3db117ae6 2024-11-11T13:35:16,104 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3a656b7bdc864291a3833ed3db117ae6, entries=150, sequenceid=222, filesize=11.9 K 2024-11-11T13:35:16,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/4406a0b68e8d444c95612ff0117c926b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/4406a0b68e8d444c95612ff0117c926b 2024-11-11T13:35:16,114 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/4406a0b68e8d444c95612ff0117c926b, entries=150, sequenceid=222, filesize=11.9 K 2024-11-11T13:35:16,120 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 3b71678035107ffe2dbde3bb5bfff436 in 324ms, sequenceid=222, compaction requested=false 2024-11-11T13:35:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:16,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T13:35:16,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T13:35:16,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T13:35:16,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-11-11T13:35:16,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.6520 sec 2024-11-11T13:35:16,192 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:16,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T13:35:16,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/17a7074fd04b4295acc69979d4b07514 is 50, key is test_row_0/A:col10/1731332116186/Put/seqid=0 2024-11-11T13:35:16,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741882_1058 (size=12151) 2024-11-11T13:35:16,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332176274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332176276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332176276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332176277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332176280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332176392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332176400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332176400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332176400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332176400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332176611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332176610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332176611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332176620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332176620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T13:35:16,658 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:16,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/17a7074fd04b4295acc69979d4b07514 2024-11-11T13:35:16,659 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T13:35:16,666 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:16,671 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:16,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T13:35:16,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/3e666d90327245e989393ff13256143a is 50, key is test_row_0/B:col10/1731332116186/Put/seqid=0 2024-11-11T13:35:16,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741883_1059 (size=12151) 2024-11-11T13:35:16,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/3e666d90327245e989393ff13256143a 2024-11-11T13:35:16,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/c3568dafd27c46fda573831a7b962938 is 50, key is test_row_0/C:col10/1731332116186/Put/seqid=0 2024-11-11T13:35:16,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741884_1060 (size=12151) 2024-11-11T13:35:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T13:35:16,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T13:35:16,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:16,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:16,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:16,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:16,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:16,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332176930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332176929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332176931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332176931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332176940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:16,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T13:35:16,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:16,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:16,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:16,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:16,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T13:35:17,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T13:35:17,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:17,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:17,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:17,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:17,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:17,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/c3568dafd27c46fda573831a7b962938 2024-11-11T13:35:17,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/17a7074fd04b4295acc69979d4b07514 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/17a7074fd04b4295acc69979d4b07514 2024-11-11T13:35:17,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/17a7074fd04b4295acc69979d4b07514, entries=150, sequenceid=235, filesize=11.9 K 2024-11-11T13:35:17,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/3e666d90327245e989393ff13256143a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3e666d90327245e989393ff13256143a 2024-11-11T13:35:17,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3e666d90327245e989393ff13256143a, entries=150, sequenceid=235, filesize=11.9 K 2024-11-11T13:35:17,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/c3568dafd27c46fda573831a7b962938 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c3568dafd27c46fda573831a7b962938 2024-11-11T13:35:17,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c3568dafd27c46fda573831a7b962938, entries=150, sequenceid=235, filesize=11.9 K 2024-11-11T13:35:17,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3b71678035107ffe2dbde3bb5bfff436 in 1051ms, sequenceid=235, compaction requested=true 2024-11-11T13:35:17,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:17,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:17,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:17,244 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:17,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:17,244 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:17,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:17,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:17,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:17,245 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:17,245 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:17,245 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/B is initiating minor compaction (all files) 2024-11-11T13:35:17,245 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/A is initiating minor compaction (all files) 2024-11-11T13:35:17,245 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/B in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:17,245 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/A in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:17,246 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/cb4d57be9f7d4394a608f0ddf1ea0af6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3a656b7bdc864291a3833ed3db117ae6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3e666d90327245e989393ff13256143a] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.0 K 2024-11-11T13:35:17,246 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/e205e7716181493f9a8c5b4a6d78144a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f3fdef481c3e4ec483eaba9860806e17, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/17a7074fd04b4295acc69979d4b07514] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.0 K 2024-11-11T13:35:17,246 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting e205e7716181493f9a8c5b4a6d78144a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731332114301 2024-11-11T13:35:17,246 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting cb4d57be9f7d4394a608f0ddf1ea0af6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731332114301 2024-11-11T13:35:17,247 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a656b7bdc864291a3833ed3db117ae6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731332114974 2024-11-11T13:35:17,247 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3fdef481c3e4ec483eaba9860806e17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731332114974 2024-11-11T13:35:17,247 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e666d90327245e989393ff13256143a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731332116185 2024-11-11T13:35:17,247 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17a7074fd04b4295acc69979d4b07514, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731332116185 2024-11-11T13:35:17,275 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:17,275 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/2afab3e8ee214337832f391c1df792e3 is 50, key is test_row_0/A:col10/1731332116186/Put/seqid=0 2024-11-11T13:35:17,281 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#B#compaction#48 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:17,281 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/a368f480570f4cb3a40ee2e54b23c866 is 50, key is test_row_0/B:col10/1731332116186/Put/seqid=0 2024-11-11T13:35:17,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T13:35:17,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:17,293 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T13:35:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T13:35:17,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/6a64ba0d00bd41c0a9c80c80231fe9c1 is 50, key is test_row_0/A:col10/1731332116275/Put/seqid=0 2024-11-11T13:35:17,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741885_1061 (size=12697) 2024-11-11T13:35:17,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741886_1062 (size=12697) 2024-11-11T13:35:17,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741887_1063 (size=12251) 2024-11-11T13:35:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:17,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:17,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332177455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332177453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332177464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332177465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332177474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332177579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332177579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332177583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332177590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332177590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,736 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/a368f480570f4cb3a40ee2e54b23c866 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/a368f480570f4cb3a40ee2e54b23c866 2024-11-11T13:35:17,743 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/2afab3e8ee214337832f391c1df792e3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/2afab3e8ee214337832f391c1df792e3 2024-11-11T13:35:17,745 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/6a64ba0d00bd41c0a9c80c80231fe9c1 2024-11-11T13:35:17,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/2f5ca62c815e444382fe84d37b836664 is 50, key is test_row_0/B:col10/1731332116275/Put/seqid=0 2024-11-11T13:35:17,768 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/A of 3b71678035107ffe2dbde3bb5bfff436 into 2afab3e8ee214337832f391c1df792e3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:17,768 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:17,768 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/A, priority=13, startTime=1731332117243; duration=0sec 2024-11-11T13:35:17,768 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:17,768 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:17,768 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:17,769 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/B of 3b71678035107ffe2dbde3bb5bfff436 into a368f480570f4cb3a40ee2e54b23c866(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:17,769 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:17,769 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/B, priority=13, startTime=1731332117244; duration=0sec 2024-11-11T13:35:17,770 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:17,770 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:17,771 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:17,771 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/C is initiating minor compaction (all files) 2024-11-11T13:35:17,771 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/C in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:17,771 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/dd6513afb476422297a9bf34664059e6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/4406a0b68e8d444c95612ff0117c926b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c3568dafd27c46fda573831a7b962938] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.0 K 2024-11-11T13:35:17,772 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd6513afb476422297a9bf34664059e6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731332114301 2024-11-11T13:35:17,773 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4406a0b68e8d444c95612ff0117c926b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731332114974 2024-11-11T13:35:17,773 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3568dafd27c46fda573831a7b962938, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731332116185 2024-11-11T13:35:17,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332177790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332177795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,798 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#C#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741888_1064 (size=12251) 2024-11-11T13:35:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332177795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332177799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,800 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/017714e7f2854e3cb66d12a7d31fdd26 is 50, key is test_row_0/C:col10/1731332116186/Put/seqid=0 2024-11-11T13:35:17,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332177800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:17,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:17,806 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/2f5ca62c815e444382fe84d37b836664 2024-11-11T13:35:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T13:35:17,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/e6ca1080eb6840548e758350930a3e65 is 50, key is test_row_0/C:col10/1731332116275/Put/seqid=0 2024-11-11T13:35:17,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741889_1065 (size=12697) 2024-11-11T13:35:17,888 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/017714e7f2854e3cb66d12a7d31fdd26 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/017714e7f2854e3cb66d12a7d31fdd26 2024-11-11T13:35:17,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741890_1066 (size=12251) 2024-11-11T13:35:17,908 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/e6ca1080eb6840548e758350930a3e65 2024-11-11T13:35:17,919 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/C of 3b71678035107ffe2dbde3bb5bfff436 into 017714e7f2854e3cb66d12a7d31fdd26(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:17,919 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:17,919 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/C, priority=13, startTime=1731332117244; duration=0sec 2024-11-11T13:35:17,920 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:17,920 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:17,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/6a64ba0d00bd41c0a9c80c80231fe9c1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6a64ba0d00bd41c0a9c80c80231fe9c1 2024-11-11T13:35:17,955 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6a64ba0d00bd41c0a9c80c80231fe9c1, entries=150, sequenceid=259, filesize=12.0 K 2024-11-11T13:35:17,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/2f5ca62c815e444382fe84d37b836664 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2f5ca62c815e444382fe84d37b836664 2024-11-11T13:35:17,965 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2f5ca62c815e444382fe84d37b836664, entries=150, sequenceid=259, filesize=12.0 K 2024-11-11T13:35:17,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/e6ca1080eb6840548e758350930a3e65 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e6ca1080eb6840548e758350930a3e65 2024-11-11T13:35:17,974 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e6ca1080eb6840548e758350930a3e65, entries=150, sequenceid=259, filesize=12.0 K 2024-11-11T13:35:17,976 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3b71678035107ffe2dbde3bb5bfff436 in 683ms, sequenceid=259, compaction requested=false 2024-11-11T13:35:17,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:17,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:17,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T13:35:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T13:35:17,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T13:35:17,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3070 sec 2024-11-11T13:35:17,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.3230 sec 2024-11-11T13:35:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:18,102 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:18,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T13:35:18,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/86f629db82d849269c32fba05a36483b is 50, key is test_row_0/A:col10/1731332117458/Put/seqid=0 2024-11-11T13:35:18,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741891_1067 (size=14741) 2024-11-11T13:35:18,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/86f629db82d849269c32fba05a36483b 2024-11-11T13:35:18,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/abbe862a5c104fa4bc7d06734d290e55 is 50, key is test_row_0/B:col10/1731332117458/Put/seqid=0 2024-11-11T13:35:18,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332178152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332178155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741892_1068 (size=12301) 2024-11-11T13:35:18,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/abbe862a5c104fa4bc7d06734d290e55 2024-11-11T13:35:18,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332178160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332178161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332178159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/dfc497d280e049a1aa0f944389c5f955 is 50, key is test_row_0/C:col10/1731332117458/Put/seqid=0 2024-11-11T13:35:18,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741893_1069 (size=12301) 2024-11-11T13:35:18,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/dfc497d280e049a1aa0f944389c5f955 2024-11-11T13:35:18,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/86f629db82d849269c32fba05a36483b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/86f629db82d849269c32fba05a36483b 2024-11-11T13:35:18,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/86f629db82d849269c32fba05a36483b, entries=200, sequenceid=275, filesize=14.4 K 2024-11-11T13:35:18,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/abbe862a5c104fa4bc7d06734d290e55 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/abbe862a5c104fa4bc7d06734d290e55 2024-11-11T13:35:18,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/abbe862a5c104fa4bc7d06734d290e55, entries=150, sequenceid=275, filesize=12.0 K 2024-11-11T13:35:18,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/dfc497d280e049a1aa0f944389c5f955 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/dfc497d280e049a1aa0f944389c5f955 2024-11-11T13:35:18,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/dfc497d280e049a1aa0f944389c5f955, entries=150, sequenceid=275, filesize=12.0 K 2024-11-11T13:35:18,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 3b71678035107ffe2dbde3bb5bfff436 in 128ms, sequenceid=275, compaction requested=true 2024-11-11T13:35:18,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:18,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:18,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:18,230 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:18,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:18,231 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:18,232 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:18,232 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:18,232 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/A is initiating minor compaction (all files) 2024-11-11T13:35:18,232 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/B is initiating minor compaction (all files) 2024-11-11T13:35:18,232 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/A in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:18,232 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/B in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:18,233 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/a368f480570f4cb3a40ee2e54b23c866, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2f5ca62c815e444382fe84d37b836664, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/abbe862a5c104fa4bc7d06734d290e55] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.4 K 2024-11-11T13:35:18,233 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/2afab3e8ee214337832f391c1df792e3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6a64ba0d00bd41c0a9c80c80231fe9c1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/86f629db82d849269c32fba05a36483b] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=38.8 K 2024-11-11T13:35:18,233 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting a368f480570f4cb3a40ee2e54b23c866, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731332116185 2024-11-11T13:35:18,233 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2afab3e8ee214337832f391c1df792e3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731332116185 2024-11-11T13:35:18,234 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f5ca62c815e444382fe84d37b836664, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731332116272 2024-11-11T13:35:18,235 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a64ba0d00bd41c0a9c80c80231fe9c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731332116272 2024-11-11T13:35:18,235 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting abbe862a5c104fa4bc7d06734d290e55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731332117456 2024-11-11T13:35:18,235 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86f629db82d849269c32fba05a36483b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731332117456 2024-11-11T13:35:18,261 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#B#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:18,262 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/ad81ea47754a4076b9ac566c656a3867 is 50, key is test_row_0/B:col10/1731332117458/Put/seqid=0 2024-11-11T13:35:18,270 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:18,271 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/19cb082aea2b465592eb7388b084a5f2 is 50, key is test_row_0/A:col10/1731332117458/Put/seqid=0 2024-11-11T13:35:18,283 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:18,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T13:35:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:18,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332178296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332178299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741894_1070 (size=12949) 2024-11-11T13:35:18,315 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/ad81ea47754a4076b9ac566c656a3867 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/ad81ea47754a4076b9ac566c656a3867 2024-11-11T13:35:18,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332178304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332178305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332178311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/036648c7befb4c1ba205a08b626895fb is 50, key is test_row_0/A:col10/1731332118272/Put/seqid=0 2024-11-11T13:35:18,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741895_1071 (size=12949) 2024-11-11T13:35:18,348 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/B of 3b71678035107ffe2dbde3bb5bfff436 into ad81ea47754a4076b9ac566c656a3867(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:18,348 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:18,348 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/B, priority=13, startTime=1731332118230; duration=0sec 2024-11-11T13:35:18,348 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:18,348 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:18,348 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:18,350 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:18,350 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/C is initiating minor compaction (all files) 2024-11-11T13:35:18,350 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/C in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:18,351 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/017714e7f2854e3cb66d12a7d31fdd26, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e6ca1080eb6840548e758350930a3e65, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/dfc497d280e049a1aa0f944389c5f955] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.4 K 2024-11-11T13:35:18,351 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 017714e7f2854e3cb66d12a7d31fdd26, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731332116185 2024-11-11T13:35:18,352 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ca1080eb6840548e758350930a3e65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731332116272 2024-11-11T13:35:18,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741896_1072 (size=17181) 2024-11-11T13:35:18,353 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting dfc497d280e049a1aa0f944389c5f955, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731332117456 2024-11-11T13:35:18,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/036648c7befb4c1ba205a08b626895fb 2024-11-11T13:35:18,362 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#C#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:18,363 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/0dfb7c43ee724ab5b6cc68a5cc1731c3 is 50, key is test_row_0/C:col10/1731332117458/Put/seqid=0 2024-11-11T13:35:18,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741897_1073 (size=12949) 2024-11-11T13:35:18,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/2e2c8888af71433cb68bdcdb9009e4a3 is 50, key is test_row_0/B:col10/1731332118272/Put/seqid=0 2024-11-11T13:35:18,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741898_1074 (size=12301) 2024-11-11T13:35:18,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/2e2c8888af71433cb68bdcdb9009e4a3 2024-11-11T13:35:18,387 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/0dfb7c43ee724ab5b6cc68a5cc1731c3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/0dfb7c43ee724ab5b6cc68a5cc1731c3 2024-11-11T13:35:18,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/1eec40e3c4724184a00a7f8d645f878a is 50, key is test_row_0/C:col10/1731332118272/Put/seqid=0 2024-11-11T13:35:18,405 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/C of 3b71678035107ffe2dbde3bb5bfff436 into 0dfb7c43ee724ab5b6cc68a5cc1731c3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:18,406 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:18,406 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/C, priority=13, startTime=1731332118231; duration=0sec 2024-11-11T13:35:18,406 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:18,406 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:18,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332178409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332178409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741899_1075 (size=12301) 2024-11-11T13:35:18,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/1eec40e3c4724184a00a7f8d645f878a 2024-11-11T13:35:18,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332178430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332178430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332178430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/036648c7befb4c1ba205a08b626895fb as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/036648c7befb4c1ba205a08b626895fb 2024-11-11T13:35:18,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/036648c7befb4c1ba205a08b626895fb, entries=250, sequenceid=302, filesize=16.8 K 2024-11-11T13:35:18,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/2e2c8888af71433cb68bdcdb9009e4a3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2e2c8888af71433cb68bdcdb9009e4a3 2024-11-11T13:35:18,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2e2c8888af71433cb68bdcdb9009e4a3, entries=150, sequenceid=302, filesize=12.0 K 2024-11-11T13:35:18,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/1eec40e3c4724184a00a7f8d645f878a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1eec40e3c4724184a00a7f8d645f878a 2024-11-11T13:35:18,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1eec40e3c4724184a00a7f8d645f878a, entries=150, sequenceid=302, filesize=12.0 K 2024-11-11T13:35:18,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 3b71678035107ffe2dbde3bb5bfff436 in 229ms, sequenceid=302, compaction requested=false 2024-11-11T13:35:18,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:18,633 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:18,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:35:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:18,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/fdc3f3d30a544e6b883607e8283f064d is 50, key is test_row_0/A:col10/1731332118294/Put/seqid=0 2024-11-11T13:35:18,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741900_1076 (size=14741) 2024-11-11T13:35:18,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/fdc3f3d30a544e6b883607e8283f064d 2024-11-11T13:35:18,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/e00e0c3e045e4e7eb29a0abd33d05b24 is 50, key is test_row_0/B:col10/1731332118294/Put/seqid=0 2024-11-11T13:35:18,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332178696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332178697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332178699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332178701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332178702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741901_1077 (size=12301) 2024-11-11T13:35:18,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/e00e0c3e045e4e7eb29a0abd33d05b24 2024-11-11T13:35:18,766 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/19cb082aea2b465592eb7388b084a5f2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/19cb082aea2b465592eb7388b084a5f2 2024-11-11T13:35:18,777 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/A of 3b71678035107ffe2dbde3bb5bfff436 into 19cb082aea2b465592eb7388b084a5f2(size=12.6 K), total size for store is 29.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:18,777 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:18,777 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/A, priority=13, startTime=1731332118230; duration=0sec 2024-11-11T13:35:18,778 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:18,778 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:18,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/a4c631fcee3945ce8ab2174cd7911e69 is 50, key is test_row_0/C:col10/1731332118294/Put/seqid=0 2024-11-11T13:35:18,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741902_1078 (size=12301) 2024-11-11T13:35:18,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332178813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332178813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332178813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332178813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332178814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T13:35:18,820 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:18,821 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T13:35:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T13:35:18,823 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:18,825 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:18,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T13:35:18,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T13:35:18,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:18,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:18,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:18,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:18,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:19,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332179022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332179022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332179030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332179030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332179030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T13:35:19,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T13:35:19,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:19,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:19,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:19,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:19,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/a4c631fcee3945ce8ab2174cd7911e69 2024-11-11T13:35:19,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/fdc3f3d30a544e6b883607e8283f064d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/fdc3f3d30a544e6b883607e8283f064d 2024-11-11T13:35:19,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/fdc3f3d30a544e6b883607e8283f064d, entries=200, sequenceid=316, filesize=14.4 K 2024-11-11T13:35:19,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/e00e0c3e045e4e7eb29a0abd33d05b24 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/e00e0c3e045e4e7eb29a0abd33d05b24 2024-11-11T13:35:19,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/e00e0c3e045e4e7eb29a0abd33d05b24, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T13:35:19,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/a4c631fcee3945ce8ab2174cd7911e69 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/a4c631fcee3945ce8ab2174cd7911e69 2024-11-11T13:35:19,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/a4c631fcee3945ce8ab2174cd7911e69, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T13:35:19,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3b71678035107ffe2dbde3bb5bfff436 in 625ms, sequenceid=316, compaction requested=true 2024-11-11T13:35:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:19,259 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:19,259 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:19,263 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:19,263 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/B is initiating minor compaction (all files) 2024-11-11T13:35:19,263 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/B in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:19,264 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/ad81ea47754a4076b9ac566c656a3867, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2e2c8888af71433cb68bdcdb9009e4a3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/e00e0c3e045e4e7eb29a0abd33d05b24] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.7 K 2024-11-11T13:35:19,264 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44871 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:19,264 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/A is initiating minor compaction (all files) 2024-11-11T13:35:19,264 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/A in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:19,264 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/19cb082aea2b465592eb7388b084a5f2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/036648c7befb4c1ba205a08b626895fb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/fdc3f3d30a544e6b883607e8283f064d] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=43.8 K 2024-11-11T13:35:19,265 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting ad81ea47754a4076b9ac566c656a3867, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731332117456 2024-11-11T13:35:19,265 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19cb082aea2b465592eb7388b084a5f2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731332117456 2024-11-11T13:35:19,265 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e2c8888af71433cb68bdcdb9009e4a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731332118149 2024-11-11T13:35:19,265 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 036648c7befb4c1ba205a08b626895fb, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731332118143 2024-11-11T13:35:19,267 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e00e0c3e045e4e7eb29a0abd33d05b24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332118294 2024-11-11T13:35:19,268 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc3f3d30a544e6b883607e8283f064d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332118294 2024-11-11T13:35:19,297 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:19,298 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/dfa01c973d1d4a618041d882250b25a7 is 50, key is test_row_0/B:col10/1731332118294/Put/seqid=0 2024-11-11T13:35:19,304 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:19,305 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/bb82c2a93220437c948e4852a1c6d8aa is 50, key is test_row_0/A:col10/1731332118294/Put/seqid=0 2024-11-11T13:35:19,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T13:35:19,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:19,308 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T13:35:19,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741904_1080 (size=13051) 2024-11-11T13:35:19,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/6d671a78cdb94b28a564b33f8c8108e0 is 50, key is test_row_0/A:col10/1731332118699/Put/seqid=0 2024-11-11T13:35:19,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741903_1079 (size=13051) 2024-11-11T13:35:19,331 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/bb82c2a93220437c948e4852a1c6d8aa as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/bb82c2a93220437c948e4852a1c6d8aa 2024-11-11T13:35:19,333 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/dfa01c973d1d4a618041d882250b25a7 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/dfa01c973d1d4a618041d882250b25a7 2024-11-11T13:35:19,342 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/A of 3b71678035107ffe2dbde3bb5bfff436 into bb82c2a93220437c948e4852a1c6d8aa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:19,342 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:19,342 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/A, priority=13, startTime=1731332119259; duration=0sec 2024-11-11T13:35:19,342 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:19,342 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:19,342 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:19,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:19,346 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/B of 3b71678035107ffe2dbde3bb5bfff436 into dfa01c973d1d4a618041d882250b25a7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:19,346 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:19,346 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/B, priority=13, startTime=1731332119259; duration=0sec 2024-11-11T13:35:19,347 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:19,347 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:19,347 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:19,347 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/C is initiating minor compaction (all files) 2024-11-11T13:35:19,347 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/C in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:19,347 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/0dfb7c43ee724ab5b6cc68a5cc1731c3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1eec40e3c4724184a00a7f8d645f878a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/a4c631fcee3945ce8ab2174cd7911e69] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.7 K 2024-11-11T13:35:19,349 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dfb7c43ee724ab5b6cc68a5cc1731c3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731332117456 2024-11-11T13:35:19,350 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eec40e3c4724184a00a7f8d645f878a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731332118149 2024-11-11T13:35:19,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741905_1081 (size=12301) 2024-11-11T13:35:19,360 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4c631fcee3945ce8ab2174cd7911e69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332118294 2024-11-11T13:35:19,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332179362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332179363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332179364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332179371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332179372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,383 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:19,384 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/27e0f100109b407fa290533c6c6ac5db is 50, key is test_row_0/C:col10/1731332118294/Put/seqid=0 2024-11-11T13:35:19,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741906_1082 (size=13051) 2024-11-11T13:35:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T13:35:19,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332179480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332179480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332179481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332179482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332179482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332179696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332179697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332179697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332179697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:19,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332179697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:19,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:19,755 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/6d671a78cdb94b28a564b33f8c8108e0 2024-11-11T13:35:19,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/a204a4e287dd4ef2b0dce656d9cb5cba is 50, key is test_row_0/B:col10/1731332118699/Put/seqid=0 2024-11-11T13:35:19,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741907_1083 (size=12301) 2024-11-11T13:35:19,788 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/a204a4e287dd4ef2b0dce656d9cb5cba 2024-11-11T13:35:19,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/7c1ae76795074640af38179c229abb79 is 50, key is test_row_0/C:col10/1731332118699/Put/seqid=0 2024-11-11T13:35:19,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741908_1084 (size=12301) 2024-11-11T13:35:19,834 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/7c1ae76795074640af38179c229abb79 2024-11-11T13:35:19,852 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/27e0f100109b407fa290533c6c6ac5db as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/27e0f100109b407fa290533c6c6ac5db 2024-11-11T13:35:19,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/6d671a78cdb94b28a564b33f8c8108e0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6d671a78cdb94b28a564b33f8c8108e0 2024-11-11T13:35:19,861 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6d671a78cdb94b28a564b33f8c8108e0, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T13:35:19,864 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/C of 3b71678035107ffe2dbde3bb5bfff436 into 27e0f100109b407fa290533c6c6ac5db(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:19,864 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:19,864 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/C, priority=13, startTime=1731332119259; duration=0sec 2024-11-11T13:35:19,865 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:19,865 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:19,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/a204a4e287dd4ef2b0dce656d9cb5cba as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/a204a4e287dd4ef2b0dce656d9cb5cba 2024-11-11T13:35:19,878 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/a204a4e287dd4ef2b0dce656d9cb5cba, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T13:35:19,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/7c1ae76795074640af38179c229abb79 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/7c1ae76795074640af38179c229abb79 2024-11-11T13:35:19,900 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/7c1ae76795074640af38179c229abb79, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T13:35:19,908 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3b71678035107ffe2dbde3bb5bfff436 in 600ms, sequenceid=341, compaction requested=false 2024-11-11T13:35:19,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:19,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:19,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T13:35:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T13:35:19,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T13:35:19,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0760 sec 2024-11-11T13:35:19,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0930 sec 2024-11-11T13:35:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T13:35:19,960 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:19,961 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T13:35:19,965 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T13:35:19,967 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:19,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:20,005 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:20,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T13:35:20,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/c2910900ca594b5e8fc45fee4165946d is 50, key is test_row_0/A:col10/1731332120004/Put/seqid=0 2024-11-11T13:35:20,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741909_1085 (size=12301) 2024-11-11T13:35:20,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/c2910900ca594b5e8fc45fee4165946d 2024-11-11T13:35:20,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332180039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332180039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332180040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332180042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332180044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/f959a803b47f4f9cb52979870fcea149 is 50, key is test_row_0/B:col10/1731332120004/Put/seqid=0 2024-11-11T13:35:20,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741910_1086 (size=12301) 2024-11-11T13:35:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T13:35:20,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332180149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332180160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332180160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332180160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332180160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:20,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:20,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T13:35:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332180365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332180371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332180370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332180374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332180371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/f959a803b47f4f9cb52979870fcea149 2024-11-11T13:35:20,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/bca96cbfe13a4aeebbe6bf11837d0d86 is 50, key is test_row_0/C:col10/1731332120004/Put/seqid=0 2024-11-11T13:35:20,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741911_1087 (size=12301) 2024-11-11T13:35:20,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/bca96cbfe13a4aeebbe6bf11837d0d86 2024-11-11T13:35:20,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/c2910900ca594b5e8fc45fee4165946d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/c2910900ca594b5e8fc45fee4165946d 2024-11-11T13:35:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T13:35:20,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/c2910900ca594b5e8fc45fee4165946d, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T13:35:20,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/f959a803b47f4f9cb52979870fcea149 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f959a803b47f4f9cb52979870fcea149 2024-11-11T13:35:20,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:20,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:20,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f959a803b47f4f9cb52979870fcea149, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T13:35:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/bca96cbfe13a4aeebbe6bf11837d0d86 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/bca96cbfe13a4aeebbe6bf11837d0d86 2024-11-11T13:35:20,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/bca96cbfe13a4aeebbe6bf11837d0d86, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T13:35:20,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3b71678035107ffe2dbde3bb5bfff436 in 653ms, sequenceid=357, compaction requested=true 2024-11-11T13:35:20,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:20,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:20,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:20,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:20,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:35:20,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:20,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T13:35:20,659 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:20,659 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:20,668 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:20,669 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/A is initiating minor compaction (all files) 2024-11-11T13:35:20,669 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/A in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,669 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/bb82c2a93220437c948e4852a1c6d8aa, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6d671a78cdb94b28a564b33f8c8108e0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/c2910900ca594b5e8fc45fee4165946d] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.8 K 2024-11-11T13:35:20,669 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb82c2a93220437c948e4852a1c6d8aa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332118294 2024-11-11T13:35:20,670 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d671a78cdb94b28a564b33f8c8108e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731332118697 2024-11-11T13:35:20,670 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2910900ca594b5e8fc45fee4165946d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731332119357 2024-11-11T13:35:20,675 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:20,675 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/C is initiating minor compaction (all files) 2024-11-11T13:35:20,675 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/C in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,676 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/27e0f100109b407fa290533c6c6ac5db, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/7c1ae76795074640af38179c229abb79, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/bca96cbfe13a4aeebbe6bf11837d0d86] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.8 K 2024-11-11T13:35:20,678 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 27e0f100109b407fa290533c6c6ac5db, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332118294 2024-11-11T13:35:20,680 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c1ae76795074640af38179c229abb79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731332118697 2024-11-11T13:35:20,681 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting bca96cbfe13a4aeebbe6bf11837d0d86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731332119357 2024-11-11T13:35:20,690 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:20,691 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/a64f4a80dbff4a538b79d2b45ab26c61 is 50, key is test_row_0/A:col10/1731332120004/Put/seqid=0 2024-11-11T13:35:20,697 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#C#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:20,698 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/130024a1a79440cbb02e5a8b2dd6cfc1 is 50, key is test_row_0/C:col10/1731332120004/Put/seqid=0 2024-11-11T13:35:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:20,704 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:20,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:35:20,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f54b9ed6646149658db664b5336aff68 is 50, key is test_row_0/A:col10/1731332120701/Put/seqid=0 2024-11-11T13:35:20,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741912_1088 (size=13153) 2024-11-11T13:35:20,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741913_1089 (size=13153) 2024-11-11T13:35:20,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332180718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332180723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,747 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/130024a1a79440cbb02e5a8b2dd6cfc1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/130024a1a79440cbb02e5a8b2dd6cfc1 2024-11-11T13:35:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332180742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332180742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,757 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/C of 3b71678035107ffe2dbde3bb5bfff436 into 130024a1a79440cbb02e5a8b2dd6cfc1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332180743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,757 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:20,757 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/C, priority=13, startTime=1731332120659; duration=0sec 2024-11-11T13:35:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,764 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:20,764 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:20,764 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:20,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741914_1090 (size=17181) 2024-11-11T13:35:20,776 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:20,776 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/B is initiating minor compaction (all files) 2024-11-11T13:35:20,776 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/B in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,776 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/dfa01c973d1d4a618041d882250b25a7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/a204a4e287dd4ef2b0dce656d9cb5cba, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f959a803b47f4f9cb52979870fcea149] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.8 K 2024-11-11T13:35:20,777 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting dfa01c973d1d4a618041d882250b25a7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332118294 2024-11-11T13:35:20,777 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting a204a4e287dd4ef2b0dce656d9cb5cba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731332118697 2024-11-11T13:35:20,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:20,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:20,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,779 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f959a803b47f4f9cb52979870fcea149, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731332119357 2024-11-11T13:35:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f54b9ed6646149658db664b5336aff68 2024-11-11T13:35:20,811 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:20,812 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/9291028bfa384687ac82ac3ed4a7fe94 is 50, key is test_row_0/B:col10/1731332120004/Put/seqid=0 2024-11-11T13:35:20,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/14ebebc8ada6434983a73ee0a9ab8069 is 50, key is test_row_0/B:col10/1731332120701/Put/seqid=0 2024-11-11T13:35:20,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741915_1091 (size=13153) 2024-11-11T13:35:20,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332180851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332180861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741916_1092 (size=12301) 2024-11-11T13:35:20,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332180863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332180864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332180870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,880 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/9291028bfa384687ac82ac3ed4a7fe94 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/9291028bfa384687ac82ac3ed4a7fe94 2024-11-11T13:35:20,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/14ebebc8ada6434983a73ee0a9ab8069 2024-11-11T13:35:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:20,901 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/B of 3b71678035107ffe2dbde3bb5bfff436 into 9291028bfa384687ac82ac3ed4a7fe94(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:20,901 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:20,901 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/B, priority=13, startTime=1731332120659; duration=0sec 2024-11-11T13:35:20,901 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:20,901 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:20,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/e0368bea2abf4f77b129a3c241b75a44 is 50, key is test_row_0/C:col10/1731332120701/Put/seqid=0 2024-11-11T13:35:20,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:20,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:20,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:20,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:20,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741917_1093 (size=12301) 2024-11-11T13:35:21,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332181069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332181079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332181089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332181089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T13:35:21,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332181100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:21,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:21,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,136 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/a64f4a80dbff4a538b79d2b45ab26c61 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a64f4a80dbff4a538b79d2b45ab26c61 2024-11-11T13:35:21,144 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/A of 3b71678035107ffe2dbde3bb5bfff436 into a64f4a80dbff4a538b79d2b45ab26c61(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:21,144 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:21,144 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/A, priority=13, startTime=1731332120659; duration=0sec 2024-11-11T13:35:21,144 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:21,144 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:21,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:21,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:21,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/e0368bea2abf4f77b129a3c241b75a44 2024-11-11T13:35:21,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f54b9ed6646149658db664b5336aff68 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f54b9ed6646149658db664b5336aff68 2024-11-11T13:35:21,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f54b9ed6646149658db664b5336aff68, entries=250, sequenceid=382, filesize=16.8 K 2024-11-11T13:35:21,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332181379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/14ebebc8ada6434983a73ee0a9ab8069 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/14ebebc8ada6434983a73ee0a9ab8069 2024-11-11T13:35:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/14ebebc8ada6434983a73ee0a9ab8069, entries=150, sequenceid=382, filesize=12.0 K 2024-11-11T13:35:21,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/e0368bea2abf4f77b129a3c241b75a44 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e0368bea2abf4f77b129a3c241b75a44 2024-11-11T13:35:21,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332181389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e0368bea2abf4f77b129a3c241b75a44, entries=150, sequenceid=382, filesize=12.0 K 2024-11-11T13:35:21,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 3b71678035107ffe2dbde3bb5bfff436 in 691ms, sequenceid=382, compaction requested=false 2024-11-11T13:35:21,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:21,402 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:21,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T13:35:21,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/0259cf68df5a4c9888be2a03a91da77c is 50, key is test_row_0/A:col10/1731332120721/Put/seqid=0 2024-11-11T13:35:21,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741918_1094 (size=14741) 2024-11-11T13:35:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332181435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332181436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332181438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332181549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332181549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332181549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:21,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:21,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332181759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332181759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332181759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/0259cf68df5a4c9888be2a03a91da77c 2024-11-11T13:35:21,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/ff8ad0a6b6a34ee78fe9bd5047bfe77a is 50, key is test_row_0/B:col10/1731332120721/Put/seqid=0 2024-11-11T13:35:21,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741919_1095 (size=12301) 2024-11-11T13:35:21,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:21,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:21,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:21,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:21,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332181890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:21,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332181899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:21,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:22,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:22,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:22,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:22,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:22,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:22,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332182069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332182070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332182070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T13:35:22,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:22,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:22,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:22,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:22,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:22,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:22,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/ff8ad0a6b6a34ee78fe9bd5047bfe77a 2024-11-11T13:35:22,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/fb71f96344fb4dd8997140ef15a1ee72 is 50, key is test_row_0/C:col10/1731332120721/Put/seqid=0 2024-11-11T13:35:22,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741920_1096 (size=12301) 2024-11-11T13:35:22,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/fb71f96344fb4dd8997140ef15a1ee72 2024-11-11T13:35:22,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/0259cf68df5a4c9888be2a03a91da77c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/0259cf68df5a4c9888be2a03a91da77c 2024-11-11T13:35:22,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/0259cf68df5a4c9888be2a03a91da77c, entries=200, sequenceid=398, filesize=14.4 K 2024-11-11T13:35:22,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/ff8ad0a6b6a34ee78fe9bd5047bfe77a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/ff8ad0a6b6a34ee78fe9bd5047bfe77a 2024-11-11T13:35:22,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/ff8ad0a6b6a34ee78fe9bd5047bfe77a, entries=150, sequenceid=398, filesize=12.0 K 2024-11-11T13:35:22,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/fb71f96344fb4dd8997140ef15a1ee72 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/fb71f96344fb4dd8997140ef15a1ee72 2024-11-11T13:35:22,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/fb71f96344fb4dd8997140ef15a1ee72, entries=150, sequenceid=398, filesize=12.0 K 2024-11-11T13:35:22,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3b71678035107ffe2dbde3bb5bfff436 in 877ms, sequenceid=398, compaction requested=true 2024-11-11T13:35:22,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:22,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:22,280 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:22,280 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:22,281 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45075 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:22,281 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:22,281 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/B is initiating minor compaction (all files) 2024-11-11T13:35:22,281 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/A is initiating minor compaction (all files) 2024-11-11T13:35:22,281 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/A in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:22,281 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/B in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:22,281 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a64f4a80dbff4a538b79d2b45ab26c61, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f54b9ed6646149658db664b5336aff68, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/0259cf68df5a4c9888be2a03a91da77c] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=44.0 K 2024-11-11T13:35:22,281 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/9291028bfa384687ac82ac3ed4a7fe94, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/14ebebc8ada6434983a73ee0a9ab8069, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/ff8ad0a6b6a34ee78fe9bd5047bfe77a] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.9 K 2024-11-11T13:35:22,282 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 9291028bfa384687ac82ac3ed4a7fe94, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731332119357 2024-11-11T13:35:22,282 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a64f4a80dbff4a538b79d2b45ab26c61, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731332119357 2024-11-11T13:35:22,283 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 14ebebc8ada6434983a73ee0a9ab8069, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731332120040 2024-11-11T13:35:22,283 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f54b9ed6646149658db664b5336aff68, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731332120036 2024-11-11T13:35:22,284 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting ff8ad0a6b6a34ee78fe9bd5047bfe77a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731332120716 2024-11-11T13:35:22,284 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0259cf68df5a4c9888be2a03a91da77c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731332120716 2024-11-11T13:35:22,294 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#B#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:22,294 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:22,294 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/3fdf4d9232fc437faff20298e10edaff is 50, key is test_row_0/B:col10/1731332120721/Put/seqid=0 2024-11-11T13:35:22,294 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/a11354cd43004c6ab5f9352a8045e1eb is 50, key is test_row_0/A:col10/1731332120721/Put/seqid=0 2024-11-11T13:35:22,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741922_1098 (size=13255) 2024-11-11T13:35:22,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741921_1097 (size=13255) 2024-11-11T13:35:22,312 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/3fdf4d9232fc437faff20298e10edaff as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3fdf4d9232fc437faff20298e10edaff 2024-11-11T13:35:22,318 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/B of 3b71678035107ffe2dbde3bb5bfff436 into 3fdf4d9232fc437faff20298e10edaff(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:22,318 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:22,318 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/B, priority=13, startTime=1731332122280; duration=0sec 2024-11-11T13:35:22,318 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:22,318 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:22,318 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:22,320 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:22,320 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/C is initiating minor compaction (all files) 2024-11-11T13:35:22,320 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/C in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:22,320 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/130024a1a79440cbb02e5a8b2dd6cfc1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e0368bea2abf4f77b129a3c241b75a44, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/fb71f96344fb4dd8997140ef15a1ee72] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=36.9 K 2024-11-11T13:35:22,321 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 130024a1a79440cbb02e5a8b2dd6cfc1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731332119357 2024-11-11T13:35:22,321 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e0368bea2abf4f77b129a3c241b75a44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731332120040 2024-11-11T13:35:22,321 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting fb71f96344fb4dd8997140ef15a1ee72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731332120716 2024-11-11T13:35:22,332 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:22,333 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/6fcbad9809ee412db9826a5686c9a615 is 50, key is test_row_0/C:col10/1731332120721/Put/seqid=0 2024-11-11T13:35:22,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T13:35:22,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:22,335 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T13:35:22,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741923_1099 (size=13255) 2024-11-11T13:35:22,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/51b12ac80f654062b14cd51a68ecbb72 is 50, key is test_row_0/A:col10/1731332121435/Put/seqid=0 2024-11-11T13:35:22,355 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/6fcbad9809ee412db9826a5686c9a615 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6fcbad9809ee412db9826a5686c9a615 2024-11-11T13:35:22,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741924_1100 (size=12301) 2024-11-11T13:35:22,365 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/C of 3b71678035107ffe2dbde3bb5bfff436 into 6fcbad9809ee412db9826a5686c9a615(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:22,365 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:22,365 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/C, priority=13, startTime=1731332122280; duration=0sec 2024-11-11T13:35:22,366 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:22,366 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:22,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:22,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332182594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332182596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332182598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332182709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332182709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332182709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,714 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/a11354cd43004c6ab5f9352a8045e1eb as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a11354cd43004c6ab5f9352a8045e1eb 2024-11-11T13:35:22,722 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/A of 3b71678035107ffe2dbde3bb5bfff436 into a11354cd43004c6ab5f9352a8045e1eb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:22,723 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:22,723 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/A, priority=13, startTime=1731332122279; duration=0sec 2024-11-11T13:35:22,723 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:22,723 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:22,762 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/51b12ac80f654062b14cd51a68ecbb72 2024-11-11T13:35:22,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/c76c1c0de21c4265ad1c776bbcb53ef6 is 50, key is test_row_0/B:col10/1731332121435/Put/seqid=0 2024-11-11T13:35:22,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741925_1101 (size=12301) 2024-11-11T13:35:22,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332182909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332182909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,916 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in 3b71678035107ffe2dbde3bb5bfff436 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:35:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,916 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in 3b71678035107ffe2dbde3bb5bfff436 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:35:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332182920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332182920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332182920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:23,175 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/c76c1c0de21c4265ad1c776bbcb53ef6 2024-11-11T13:35:23,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/63dfbede026c45dcaa51cdf71958d052 is 50, key is test_row_0/C:col10/1731332121435/Put/seqid=0 2024-11-11T13:35:23,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741926_1102 (size=12301) 2024-11-11T13:35:23,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332183229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332183229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:23,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332183229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:23,589 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/63dfbede026c45dcaa51cdf71958d052 2024-11-11T13:35:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/51b12ac80f654062b14cd51a68ecbb72 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/51b12ac80f654062b14cd51a68ecbb72 2024-11-11T13:35:23,612 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/51b12ac80f654062b14cd51a68ecbb72, entries=150, sequenceid=422, filesize=12.0 K 2024-11-11T13:35:23,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/c76c1c0de21c4265ad1c776bbcb53ef6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c76c1c0de21c4265ad1c776bbcb53ef6 2024-11-11T13:35:23,620 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c76c1c0de21c4265ad1c776bbcb53ef6, entries=150, sequenceid=422, filesize=12.0 K 2024-11-11T13:35:23,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/63dfbede026c45dcaa51cdf71958d052 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/63dfbede026c45dcaa51cdf71958d052 2024-11-11T13:35:23,631 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/63dfbede026c45dcaa51cdf71958d052, entries=150, sequenceid=422, filesize=12.0 K 2024-11-11T13:35:23,633 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 3b71678035107ffe2dbde3bb5bfff436 in 1298ms, sequenceid=422, compaction requested=false 2024-11-11T13:35:23,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:23,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:23,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T13:35:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T13:35:23,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T13:35:23,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6680 sec 2024-11-11T13:35:23,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 3.6770 sec 2024-11-11T13:35:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:23,769 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:23,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T13:35:23,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/11fcd617be0448ce85a606e3eb32bd92 is 50, key is test_row_0/A:col10/1731332123741/Put/seqid=0 2024-11-11T13:35:23,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741927_1103 (size=12301) 2024-11-11T13:35:23,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:23,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332183831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332183831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332183832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:23,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332183949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:23,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332183949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:23,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332183949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:23,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:23,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T13:35:24,129 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:24,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-11T13:35:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T13:35:24,131 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:24,132 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:24,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:24,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:24,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332184159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332184159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332184159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:24,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/11fcd617be0448ce85a606e3eb32bd92 2024-11-11T13:35:24,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/576f2ba4340c43bf898cacda69670ba8 is 50, key is test_row_0/B:col10/1731332123741/Put/seqid=0 2024-11-11T13:35:24,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741928_1104 (size=12301) 2024-11-11T13:35:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T13:35:24,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T13:35:24,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:24,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:24,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:24,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T13:35:24,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:24,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:24,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:24,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T13:35:24,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332184469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:24,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332184469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:24,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332184470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:24,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T13:35:24,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:24,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/576f2ba4340c43bf898cacda69670ba8 2024-11-11T13:35:24,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/be4743337b4a45d79e6995bcad049c80 is 50, key is test_row_0/C:col10/1731332123741/Put/seqid=0 2024-11-11T13:35:24,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741929_1105 (size=12301) 2024-11-11T13:35:24,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T13:35:24,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:24,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:24,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:24,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T13:35:24,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T13:35:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:24,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:24,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41530 deadline: 1731332184919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:24,923 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in 3b71678035107ffe2dbde3bb5bfff436 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:35:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:24,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41582 deadline: 1731332184940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:24,944 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in 3b71678035107ffe2dbde3bb5bfff436 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:35:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:24,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332184979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:24,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:24,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332184979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332184980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:25,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/be4743337b4a45d79e6995bcad049c80 2024-11-11T13:35:25,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/11fcd617be0448ce85a606e3eb32bd92 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/11fcd617be0448ce85a606e3eb32bd92 2024-11-11T13:35:25,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/11fcd617be0448ce85a606e3eb32bd92, entries=150, sequenceid=440, filesize=12.0 K 2024-11-11T13:35:25,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/576f2ba4340c43bf898cacda69670ba8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/576f2ba4340c43bf898cacda69670ba8 2024-11-11T13:35:25,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/576f2ba4340c43bf898cacda69670ba8, entries=150, sequenceid=440, filesize=12.0 K 2024-11-11T13:35:25,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/be4743337b4a45d79e6995bcad049c80 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/be4743337b4a45d79e6995bcad049c80 2024-11-11T13:35:25,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/be4743337b4a45d79e6995bcad049c80, entries=150, sequenceid=440, filesize=12.0 K 2024-11-11T13:35:25,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3b71678035107ffe2dbde3bb5bfff436 in 1276ms, sequenceid=440, compaction requested=true 2024-11-11T13:35:25,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:25,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:25,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:25,046 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:25,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:25,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:25,046 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:25,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:25,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:25,047 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:25,047 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:25,047 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/B is initiating minor compaction (all files) 2024-11-11T13:35:25,047 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/A is initiating minor compaction (all files) 2024-11-11T13:35:25,047 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/B in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:25,047 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/A in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:25,047 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3fdf4d9232fc437faff20298e10edaff, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c76c1c0de21c4265ad1c776bbcb53ef6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/576f2ba4340c43bf898cacda69670ba8] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=37.0 K 2024-11-11T13:35:25,047 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a11354cd43004c6ab5f9352a8045e1eb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/51b12ac80f654062b14cd51a68ecbb72, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/11fcd617be0448ce85a606e3eb32bd92] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=37.0 K 2024-11-11T13:35:25,048 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fdf4d9232fc437faff20298e10edaff, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731332120716 2024-11-11T13:35:25,048 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a11354cd43004c6ab5f9352a8045e1eb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731332120716 2024-11-11T13:35:25,048 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c76c1c0de21c4265ad1c776bbcb53ef6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731332121432 2024-11-11T13:35:25,048 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51b12ac80f654062b14cd51a68ecbb72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731332121432 2024-11-11T13:35:25,048 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 576f2ba4340c43bf898cacda69670ba8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731332122594 2024-11-11T13:35:25,049 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11fcd617be0448ce85a606e3eb32bd92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731332122594 2024-11-11T13:35:25,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T13:35:25,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:25,052 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T13:35:25,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f52d6d841cbf4f4d97ba4cc43527dd36 is 50, key is test_row_0/A:col10/1731332123794/Put/seqid=0 2024-11-11T13:35:25,060 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:25,061 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/c1177776636340b2b452306215df0a7b is 50, key is test_row_0/B:col10/1731332123741/Put/seqid=0 2024-11-11T13:35:25,065 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#A#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:25,066 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/669e31cc21e6483ea2d366294d9a220d is 50, key is test_row_0/A:col10/1731332123741/Put/seqid=0 2024-11-11T13:35:25,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741932_1108 (size=13357) 2024-11-11T13:35:25,088 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/669e31cc21e6483ea2d366294d9a220d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/669e31cc21e6483ea2d366294d9a220d 2024-11-11T13:35:25,094 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/A of 3b71678035107ffe2dbde3bb5bfff436 into 669e31cc21e6483ea2d366294d9a220d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:25,094 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:25,094 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/A, priority=13, startTime=1731332125046; duration=0sec 2024-11-11T13:35:25,095 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:25,095 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:25,095 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:25,097 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:25,097 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/C is initiating minor compaction (all files) 2024-11-11T13:35:25,097 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/C in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:25,097 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6fcbad9809ee412db9826a5686c9a615, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/63dfbede026c45dcaa51cdf71958d052, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/be4743337b4a45d79e6995bcad049c80] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=37.0 K 2024-11-11T13:35:25,098 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fcbad9809ee412db9826a5686c9a615, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731332120716 2024-11-11T13:35:25,098 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63dfbede026c45dcaa51cdf71958d052, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731332121432 2024-11-11T13:35:25,099 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting be4743337b4a45d79e6995bcad049c80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731332122594 2024-11-11T13:35:25,113 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#C#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:25,114 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/59a143cae35e4a7196445938fd598699 is 50, key is test_row_0/C:col10/1731332123741/Put/seqid=0 2024-11-11T13:35:25,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741930_1106 (size=12301) 2024-11-11T13:35:25,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741931_1107 (size=13357) 2024-11-11T13:35:25,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741933_1109 (size=13357) 2024-11-11T13:35:25,140 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/59a143cae35e4a7196445938fd598699 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/59a143cae35e4a7196445938fd598699 2024-11-11T13:35:25,146 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/C of 3b71678035107ffe2dbde3bb5bfff436 into 59a143cae35e4a7196445938fd598699(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:25,146 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:25,146 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/C, priority=13, startTime=1731332125046; duration=0sec 2024-11-11T13:35:25,146 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:25,146 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T13:35:25,517 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f52d6d841cbf4f4d97ba4cc43527dd36 2024-11-11T13:35:25,528 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/c1177776636340b2b452306215df0a7b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c1177776636340b2b452306215df0a7b 2024-11-11T13:35:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/d3c5a764ee8843af9dab7dc796a826ae is 50, key is test_row_0/B:col10/1731332123794/Put/seqid=0 2024-11-11T13:35:25,535 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/B of 3b71678035107ffe2dbde3bb5bfff436 into c1177776636340b2b452306215df0a7b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:25,535 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:25,535 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/B, priority=13, startTime=1731332125046; duration=0sec 2024-11-11T13:35:25,535 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:25,535 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:25,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741934_1110 (size=12301) 2024-11-11T13:35:25,939 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/d3c5a764ee8843af9dab7dc796a826ae 2024-11-11T13:35:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/3bcf688ff85f4d2bbc0c5154811931a1 is 50, key is test_row_0/C:col10/1731332123794/Put/seqid=0 2024-11-11T13:35:25,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741935_1111 (size=12301) 2024-11-11T13:35:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:25,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. as already flushing 2024-11-11T13:35:26,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332186034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332186035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332186036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332186149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332186149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332186150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T13:35:26,355 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/3bcf688ff85f4d2bbc0c5154811931a1 2024-11-11T13:35:26,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/f52d6d841cbf4f4d97ba4cc43527dd36 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f52d6d841cbf4f4d97ba4cc43527dd36 2024-11-11T13:35:26,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332186359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332186360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332186360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,366 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f52d6d841cbf4f4d97ba4cc43527dd36, entries=150, sequenceid=460, filesize=12.0 K 2024-11-11T13:35:26,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/d3c5a764ee8843af9dab7dc796a826ae as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/d3c5a764ee8843af9dab7dc796a826ae 2024-11-11T13:35:26,372 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/d3c5a764ee8843af9dab7dc796a826ae, entries=150, sequenceid=460, filesize=12.0 K 2024-11-11T13:35:26,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/3bcf688ff85f4d2bbc0c5154811931a1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/3bcf688ff85f4d2bbc0c5154811931a1 2024-11-11T13:35:26,379 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/3bcf688ff85f4d2bbc0c5154811931a1, entries=150, sequenceid=460, filesize=12.0 K 2024-11-11T13:35:26,381 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 3b71678035107ffe2dbde3bb5bfff436 in 1329ms, sequenceid=460, compaction requested=false 2024-11-11T13:35:26,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:26,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:26,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-11T13:35:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-11T13:35:26,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T13:35:26,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2500 sec 2024-11-11T13:35:26,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.2550 sec 2024-11-11T13:35:26,671 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:26,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-11T13:35:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:26,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/933c059241d34ef7b8a4d23a68e15856 is 50, key is test_row_0/A:col10/1731332126035/Put/seqid=0 2024-11-11T13:35:26,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741936_1112 (size=12301) 2024-11-11T13:35:26,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332186701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332186703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332186705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332186809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332186809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:26,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332186819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332187019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332187019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332187029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/933c059241d34ef7b8a4d23a68e15856 2024-11-11T13:35:27,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/437d6c11094c4bb3a5fff1402d2d3023 is 50, key is test_row_0/B:col10/1731332126035/Put/seqid=0 2024-11-11T13:35:27,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741937_1113 (size=12301) 2024-11-11T13:35:27,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/437d6c11094c4bb3a5fff1402d2d3023 2024-11-11T13:35:27,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/e1555444b9564024b0728eb261c1a74c is 50, key is test_row_0/C:col10/1731332126035/Put/seqid=0 2024-11-11T13:35:27,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741938_1114 (size=12301) 2024-11-11T13:35:27,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/e1555444b9564024b0728eb261c1a74c 2024-11-11T13:35:27,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/933c059241d34ef7b8a4d23a68e15856 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/933c059241d34ef7b8a4d23a68e15856 2024-11-11T13:35:27,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/933c059241d34ef7b8a4d23a68e15856, entries=150, sequenceid=481, filesize=12.0 K 2024-11-11T13:35:27,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/437d6c11094c4bb3a5fff1402d2d3023 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/437d6c11094c4bb3a5fff1402d2d3023 2024-11-11T13:35:27,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/437d6c11094c4bb3a5fff1402d2d3023, entries=150, sequenceid=481, filesize=12.0 K 2024-11-11T13:35:27,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/e1555444b9564024b0728eb261c1a74c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e1555444b9564024b0728eb261c1a74c 2024-11-11T13:35:27,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e1555444b9564024b0728eb261c1a74c, entries=150, sequenceid=481, filesize=12.0 K 2024-11-11T13:35:27,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 3b71678035107ffe2dbde3bb5bfff436 in 534ms, sequenceid=481, compaction requested=true 2024-11-11T13:35:27,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:27,206 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:27,207 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:27,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:27,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:27,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:27,211 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:27,211 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/A is initiating minor compaction (all files) 2024-11-11T13:35:27,211 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/A in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:27,211 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/669e31cc21e6483ea2d366294d9a220d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f52d6d841cbf4f4d97ba4cc43527dd36, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/933c059241d34ef7b8a4d23a68e15856] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=37.1 K 2024-11-11T13:35:27,212 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:27,212 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 669e31cc21e6483ea2d366294d9a220d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731332122594 2024-11-11T13:35:27,212 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/B is initiating minor compaction (all files) 2024-11-11T13:35:27,213 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/B in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:27,213 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c1177776636340b2b452306215df0a7b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/d3c5a764ee8843af9dab7dc796a826ae, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/437d6c11094c4bb3a5fff1402d2d3023] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=37.1 K 2024-11-11T13:35:27,213 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f52d6d841cbf4f4d97ba4cc43527dd36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731332123794 2024-11-11T13:35:27,213 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c1177776636340b2b452306215df0a7b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731332122594 2024-11-11T13:35:27,213 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 933c059241d34ef7b8a4d23a68e15856, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731332126026 2024-11-11T13:35:27,214 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting d3c5a764ee8843af9dab7dc796a826ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731332123794 2024-11-11T13:35:27,216 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 437d6c11094c4bb3a5fff1402d2d3023, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731332126026 2024-11-11T13:35:27,230 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#B#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:27,231 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/043ef00548f44a94b16ba045a86de5a7 is 50, key is test_row_0/B:col10/1731332126035/Put/seqid=0 2024-11-11T13:35:27,241 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#A#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:27,242 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/149e4960779a44c493c2f90b664570dc is 50, key is test_row_0/A:col10/1731332126035/Put/seqid=0 2024-11-11T13:35:27,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741939_1115 (size=13459) 2024-11-11T13:35:27,264 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/043ef00548f44a94b16ba045a86de5a7 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/043ef00548f44a94b16ba045a86de5a7 2024-11-11T13:35:27,277 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/B of 3b71678035107ffe2dbde3bb5bfff436 into 043ef00548f44a94b16ba045a86de5a7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:27,277 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:27,277 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/B, priority=13, startTime=1731332127207; duration=0sec 2024-11-11T13:35:27,277 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:27,277 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:27,277 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:27,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741940_1116 (size=13459) 2024-11-11T13:35:27,279 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:27,279 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 3b71678035107ffe2dbde3bb5bfff436/C is initiating minor compaction (all files) 2024-11-11T13:35:27,279 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3b71678035107ffe2dbde3bb5bfff436/C in TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:27,280 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/59a143cae35e4a7196445938fd598699, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/3bcf688ff85f4d2bbc0c5154811931a1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e1555444b9564024b0728eb261c1a74c] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp, totalSize=37.1 K 2024-11-11T13:35:27,281 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a143cae35e4a7196445938fd598699, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731332122594 2024-11-11T13:35:27,281 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bcf688ff85f4d2bbc0c5154811931a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731332123794 2024-11-11T13:35:27,281 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e1555444b9564024b0728eb261c1a74c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1731332126026 2024-11-11T13:35:27,284 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/149e4960779a44c493c2f90b664570dc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/149e4960779a44c493c2f90b664570dc 2024-11-11T13:35:27,289 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3b71678035107ffe2dbde3bb5bfff436#C#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:27,290 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/736118cff3f44aa18a5566a481300c37 is 50, key is test_row_0/C:col10/1731332126035/Put/seqid=0 2024-11-11T13:35:27,290 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/A of 3b71678035107ffe2dbde3bb5bfff436 into 149e4960779a44c493c2f90b664570dc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:27,291 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:27,292 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/A, priority=13, startTime=1731332127206; duration=0sec 2024-11-11T13:35:27,292 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:27,292 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:27,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741941_1117 (size=13459) 2024-11-11T13:35:27,312 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/736118cff3f44aa18a5566a481300c37 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/736118cff3f44aa18a5566a481300c37 2024-11-11T13:35:27,322 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3b71678035107ffe2dbde3bb5bfff436/C of 3b71678035107ffe2dbde3bb5bfff436 into 736118cff3f44aa18a5566a481300c37(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:27,322 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:27,322 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., storeName=3b71678035107ffe2dbde3bb5bfff436/C, priority=13, startTime=1731332127208; duration=0sec 2024-11-11T13:35:27,322 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:27,322 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:27,331 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:27,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T13:35:27,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/a9067cf2e3c346ad8f11ebacaf80a04c is 50, key is test_row_0/A:col10/1731332127330/Put/seqid=0 2024-11-11T13:35:27,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741942_1118 (size=12301) 2024-11-11T13:35:27,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332187365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332187368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332187369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332187479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332187479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332187479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332187689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332187689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332187689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:27,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/a9067cf2e3c346ad8f11ebacaf80a04c 2024-11-11T13:35:27,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/0c6cebc249b944838d10ed5e836a45e6 is 50, key is test_row_0/B:col10/1731332127330/Put/seqid=0 2024-11-11T13:35:27,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741943_1119 (size=12301) 2024-11-11T13:35:27,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/0c6cebc249b944838d10ed5e836a45e6 2024-11-11T13:35:27,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/3e6eecaa33f5442f97b8816d76963a52 is 50, key is test_row_0/C:col10/1731332127330/Put/seqid=0 2024-11-11T13:35:27,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741944_1120 (size=12301) 2024-11-11T13:35:27,812 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T13:35:27,812 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:27,814 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T13:35:27,814 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:27,815 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T13:35:27,815 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:27,817 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T13:35:27,817 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:27,818 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:27,818 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:27,818 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:27,818 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:27,818 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:27,818 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:27,818 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:27,818 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:27,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:27,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:27,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:27,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:28,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:28,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41566 deadline: 1731332187999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:28,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41620 deadline: 1731332187999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41606 deadline: 1731332188000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:28,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:28,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:35:28,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/3e6eecaa33f5442f97b8816d76963a52 2024-11-11T13:35:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/a9067cf2e3c346ad8f11ebacaf80a04c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a9067cf2e3c346ad8f11ebacaf80a04c 2024-11-11T13:35:28,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a9067cf2e3c346ad8f11ebacaf80a04c, entries=150, sequenceid=504, filesize=12.0 K 2024-11-11T13:35:28,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/0c6cebc249b944838d10ed5e836a45e6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0c6cebc249b944838d10ed5e836a45e6 2024-11-11T13:35:28,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0c6cebc249b944838d10ed5e836a45e6, entries=150, sequenceid=504, filesize=12.0 K 2024-11-11T13:35:28,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/3e6eecaa33f5442f97b8816d76963a52 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/3e6eecaa33f5442f97b8816d76963a52 2024-11-11T13:35:28,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/3e6eecaa33f5442f97b8816d76963a52, entries=150, sequenceid=504, filesize=12.0 K 2024-11-11T13:35:28,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 3b71678035107ffe2dbde3bb5bfff436 in 884ms, sequenceid=504, compaction requested=false 2024-11-11T13:35:28,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T13:35:28,311 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:28,510 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:28,511 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T13:35:28,511 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T13:35:28,511 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T13:35:28,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-11T13:35:28,511 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:28,511 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:28,511 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:28,511 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,511 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,511 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,511 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,511 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,511 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:28,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:28,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:28,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/cc4745a9ae91493f90c9025ec60b062c is 50, key is test_row_0/A:col10/1731332128509/Put/seqid=0 2024-11-11T13:35:28,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741945_1121 (size=12301) 2024-11-11T13:35:28,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/cc4745a9ae91493f90c9025ec60b062c 2024-11-11T13:35:28,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/4a0f323a28e74f9aaf3b84daaa9e38fa is 50, key is test_row_0/B:col10/1731332128509/Put/seqid=0 2024-11-11T13:35:28,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741946_1122 (size=12301) 2024-11-11T13:35:28,950 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T13:35:28,950 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:28,950 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,950 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:28,982 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T13:35:28,982 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:28,982 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,983 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:28,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T13:35:28,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-11T13:35:28,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-11T13:35:28,983 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-11T13:35:28,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-11T13:35:28,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-11T13:35:28,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T13:35:28,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2929 2024-11-11T13:35:28,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2945 2024-11-11T13:35:28,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T13:35:28,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1330 2024-11-11T13:35:28,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3990 rows 2024-11-11T13:35:28,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1355 2024-11-11T13:35:28,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4056 rows 2024-11-11T13:35:28,984 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:35:28,985 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:35:28,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:28,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:28,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T13:35:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T13:35:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T13:35:28,999 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332128999"}]},"ts":"1731332128999"} 2024-11-11T13:35:29,001 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T13:35:29,001 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T13:35:29,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T13:35:29,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3b71678035107ffe2dbde3bb5bfff436, UNASSIGN}] 2024-11-11T13:35:29,007 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3b71678035107ffe2dbde3bb5bfff436, UNASSIGN 2024-11-11T13:35:29,009 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35609 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=37d26ecf6460,36877,1731332101576, table=TestAcidGuarantees, region=3b71678035107ffe2dbde3bb5bfff436. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T13:35:29,009 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=3b71678035107ffe2dbde3bb5bfff436, regionState=CLOSING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:29,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3b71678035107ffe2dbde3bb5bfff436, UNASSIGN because future has completed 2024-11-11T13:35:29,012 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T13:35:29,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:35:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T13:35:29,171 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:29,171 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T13:35:29,172 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 3b71678035107ffe2dbde3bb5bfff436, disabling compactions & flushes 2024-11-11T13:35:29,173 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T13:35:29,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/4a0f323a28e74f9aaf3b84daaa9e38fa 2024-11-11T13:35:29,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/304c7d64a22e49f08c5acf85dbd5ffe2 is 50, key is test_row_0/C:col10/1731332128509/Put/seqid=0 2024-11-11T13:35:29,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741947_1123 (size=12301) 2024-11-11T13:35:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T13:35:29,756 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T13:35:29,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/304c7d64a22e49f08c5acf85dbd5ffe2 2024-11-11T13:35:29,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/cc4745a9ae91493f90c9025ec60b062c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/cc4745a9ae91493f90c9025ec60b062c 2024-11-11T13:35:29,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/cc4745a9ae91493f90c9025ec60b062c, entries=150, sequenceid=523, filesize=12.0 K 2024-11-11T13:35:29,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/4a0f323a28e74f9aaf3b84daaa9e38fa as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/4a0f323a28e74f9aaf3b84daaa9e38fa 2024-11-11T13:35:29,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/4a0f323a28e74f9aaf3b84daaa9e38fa, entries=150, sequenceid=523, filesize=12.0 K 2024-11-11T13:35:29,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/304c7d64a22e49f08c5acf85dbd5ffe2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/304c7d64a22e49f08c5acf85dbd5ffe2 2024-11-11T13:35:29,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/304c7d64a22e49f08c5acf85dbd5ffe2, entries=150, sequenceid=523, filesize=12.0 K 2024-11-11T13:35:29,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=13.42 KB/13740 for 3b71678035107ffe2dbde3bb5bfff436 in 1282ms, sequenceid=523, compaction requested=true 2024-11-11T13:35:29,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3b71678035107ffe2dbde3bb5bfff436: 2024-11-11T13:35:29,792 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:29,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:29,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:29,792 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:29,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:29,792 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. after waiting 0 ms 2024-11-11T13:35:29,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:29,792 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:29,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3b71678035107ffe2dbde3bb5bfff436:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:29,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:29,792 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. because compaction request was cancelled 2024-11-11T13:35:29,792 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. because compaction request was cancelled 2024-11-11T13:35:29,792 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:B 2024-11-11T13:35:29,792 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 3b71678035107ffe2dbde3bb5bfff436 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-11T13:35:29,792 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:A 2024-11-11T13:35:29,792 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. because compaction request was cancelled 2024-11-11T13:35:29,792 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3b71678035107ffe2dbde3bb5bfff436:C 2024-11-11T13:35:29,796 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/9e0696ac548845788c42f9798ccd8f28 is 50, key is test_row_1/A:col10/1731332128980/Put/seqid=0 2024-11-11T13:35:29,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741948_1124 (size=7415) 2024-11-11T13:35:30,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T13:35:30,202 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/9e0696ac548845788c42f9798ccd8f28 2024-11-11T13:35:30,216 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/622f4e80096f4e72be6436ffda98398c is 50, key is test_row_1/B:col10/1731332128980/Put/seqid=0 2024-11-11T13:35:30,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741949_1125 (size=7415) 2024-11-11T13:35:30,621 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/622f4e80096f4e72be6436ffda98398c 2024-11-11T13:35:30,657 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/a59c13a6d7cf4ff98068d3bda2526463 is 50, key is test_row_1/C:col10/1731332128980/Put/seqid=0 2024-11-11T13:35:30,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741950_1126 (size=7415) 2024-11-11T13:35:31,063 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/a59c13a6d7cf4ff98068d3bda2526463 2024-11-11T13:35:31,072 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/A/9e0696ac548845788c42f9798ccd8f28 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9e0696ac548845788c42f9798ccd8f28 2024-11-11T13:35:31,078 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9e0696ac548845788c42f9798ccd8f28, entries=50, sequenceid=528, filesize=7.2 K 2024-11-11T13:35:31,079 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/B/622f4e80096f4e72be6436ffda98398c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/622f4e80096f4e72be6436ffda98398c 2024-11-11T13:35:31,085 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/622f4e80096f4e72be6436ffda98398c, entries=50, sequenceid=528, filesize=7.2 K 2024-11-11T13:35:31,086 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/.tmp/C/a59c13a6d7cf4ff98068d3bda2526463 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/a59c13a6d7cf4ff98068d3bda2526463 2024-11-11T13:35:31,091 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/a59c13a6d7cf4ff98068d3bda2526463, entries=50, sequenceid=528, filesize=7.2 K 2024-11-11T13:35:31,092 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 3b71678035107ffe2dbde3bb5bfff436 in 1300ms, sequenceid=528, compaction requested=true 2024-11-11T13:35:31,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/7d94a1098a8c414280f58bedd18487d8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f150fd84e0f84371ac94338e407d9d69, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/366d417dec3a42bdad8fbfc6ee8ec80a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9f7f444dbe7b4227942c3413c72f8f0f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/590b3ee34e694feea661d748bf37ce8a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6c075679de414f09a829276642146e96, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/4dd0490712a640f1a61cc3717da1d005, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/74e0aab0441d407caad081cf6df795db, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9ec3341bcf984171aed0424bbb2fb8bc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/2069050e8e33433ab201752012aa7304, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/06890239400c4b508cded0becde4bdb3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9149189ddbbd4634818fe72e4f6293e4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/e205e7716181493f9a8c5b4a6d78144a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/176c26a3cc354d058292ff93830f7d87, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f3fdef481c3e4ec483eaba9860806e17, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/2afab3e8ee214337832f391c1df792e3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/17a7074fd04b4295acc69979d4b07514, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6a64ba0d00bd41c0a9c80c80231fe9c1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/86f629db82d849269c32fba05a36483b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/19cb082aea2b465592eb7388b084a5f2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/036648c7befb4c1ba205a08b626895fb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/fdc3f3d30a544e6b883607e8283f064d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/bb82c2a93220437c948e4852a1c6d8aa, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6d671a78cdb94b28a564b33f8c8108e0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a64f4a80dbff4a538b79d2b45ab26c61, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/c2910900ca594b5e8fc45fee4165946d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f54b9ed6646149658db664b5336aff68, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/0259cf68df5a4c9888be2a03a91da77c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a11354cd43004c6ab5f9352a8045e1eb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/51b12ac80f654062b14cd51a68ecbb72, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/669e31cc21e6483ea2d366294d9a220d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/11fcd617be0448ce85a606e3eb32bd92, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f52d6d841cbf4f4d97ba4cc43527dd36, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/933c059241d34ef7b8a4d23a68e15856] to archive 2024-11-11T13:35:31,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:35:31,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/7d94a1098a8c414280f58bedd18487d8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/7d94a1098a8c414280f58bedd18487d8 2024-11-11T13:35:31,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f150fd84e0f84371ac94338e407d9d69 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f150fd84e0f84371ac94338e407d9d69 2024-11-11T13:35:31,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/366d417dec3a42bdad8fbfc6ee8ec80a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/366d417dec3a42bdad8fbfc6ee8ec80a 2024-11-11T13:35:31,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9f7f444dbe7b4227942c3413c72f8f0f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9f7f444dbe7b4227942c3413c72f8f0f 2024-11-11T13:35:31,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/590b3ee34e694feea661d748bf37ce8a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/590b3ee34e694feea661d748bf37ce8a 2024-11-11T13:35:31,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6c075679de414f09a829276642146e96 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6c075679de414f09a829276642146e96 2024-11-11T13:35:31,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/4dd0490712a640f1a61cc3717da1d005 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/4dd0490712a640f1a61cc3717da1d005 2024-11-11T13:35:31,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/74e0aab0441d407caad081cf6df795db to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/74e0aab0441d407caad081cf6df795db 2024-11-11T13:35:31,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9ec3341bcf984171aed0424bbb2fb8bc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9ec3341bcf984171aed0424bbb2fb8bc 2024-11-11T13:35:31,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/2069050e8e33433ab201752012aa7304 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/2069050e8e33433ab201752012aa7304 2024-11-11T13:35:31,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/06890239400c4b508cded0becde4bdb3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/06890239400c4b508cded0becde4bdb3 2024-11-11T13:35:31,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9149189ddbbd4634818fe72e4f6293e4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9149189ddbbd4634818fe72e4f6293e4 2024-11-11T13:35:31,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/e205e7716181493f9a8c5b4a6d78144a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/e205e7716181493f9a8c5b4a6d78144a 2024-11-11T13:35:31,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/176c26a3cc354d058292ff93830f7d87 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/176c26a3cc354d058292ff93830f7d87 2024-11-11T13:35:31,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f3fdef481c3e4ec483eaba9860806e17 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f3fdef481c3e4ec483eaba9860806e17 2024-11-11T13:35:31,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/2afab3e8ee214337832f391c1df792e3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/2afab3e8ee214337832f391c1df792e3 2024-11-11T13:35:31,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/17a7074fd04b4295acc69979d4b07514 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/17a7074fd04b4295acc69979d4b07514 2024-11-11T13:35:31,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6a64ba0d00bd41c0a9c80c80231fe9c1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6a64ba0d00bd41c0a9c80c80231fe9c1 2024-11-11T13:35:31,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/86f629db82d849269c32fba05a36483b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/86f629db82d849269c32fba05a36483b 2024-11-11T13:35:31,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/19cb082aea2b465592eb7388b084a5f2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/19cb082aea2b465592eb7388b084a5f2 2024-11-11T13:35:31,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/036648c7befb4c1ba205a08b626895fb to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/036648c7befb4c1ba205a08b626895fb 2024-11-11T13:35:31,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/fdc3f3d30a544e6b883607e8283f064d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/fdc3f3d30a544e6b883607e8283f064d 2024-11-11T13:35:31,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/bb82c2a93220437c948e4852a1c6d8aa to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/bb82c2a93220437c948e4852a1c6d8aa 2024-11-11T13:35:31,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6d671a78cdb94b28a564b33f8c8108e0 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/6d671a78cdb94b28a564b33f8c8108e0 2024-11-11T13:35:31,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a64f4a80dbff4a538b79d2b45ab26c61 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a64f4a80dbff4a538b79d2b45ab26c61 2024-11-11T13:35:31,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/c2910900ca594b5e8fc45fee4165946d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/c2910900ca594b5e8fc45fee4165946d 2024-11-11T13:35:31,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f54b9ed6646149658db664b5336aff68 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f54b9ed6646149658db664b5336aff68 2024-11-11T13:35:31,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/0259cf68df5a4c9888be2a03a91da77c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/0259cf68df5a4c9888be2a03a91da77c 2024-11-11T13:35:31,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a11354cd43004c6ab5f9352a8045e1eb to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a11354cd43004c6ab5f9352a8045e1eb 2024-11-11T13:35:31,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/51b12ac80f654062b14cd51a68ecbb72 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/51b12ac80f654062b14cd51a68ecbb72 2024-11-11T13:35:31,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/669e31cc21e6483ea2d366294d9a220d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/669e31cc21e6483ea2d366294d9a220d 2024-11-11T13:35:31,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/11fcd617be0448ce85a606e3eb32bd92 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/11fcd617be0448ce85a606e3eb32bd92 2024-11-11T13:35:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T13:35:31,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f52d6d841cbf4f4d97ba4cc43527dd36 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/f52d6d841cbf4f4d97ba4cc43527dd36 2024-11-11T13:35:31,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/933c059241d34ef7b8a4d23a68e15856 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/933c059241d34ef7b8a4d23a68e15856 2024-11-11T13:35:31,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f81b141dcb134e4e95ececba154cc883, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1ef5950b9f40402081d36b0fd06a5d11, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/23c560098cb746e2ad32b1ea302af89a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/4059bb934ec94f87b8ef12fed7c9d270, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c07b03fbbe5b4e09ac08e530db4c326d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1bae6447da714f40beda8c786853ed2a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/5cf5c690da274370add1892444a9ae4f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/18342b3039794313bc568d208cc042fa, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0937dc0df14947c696bbe21f458892fd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0d205f47a16344b794b48d26a2b2e7be, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/6d89665b683e4fc58b2a4798356ab7f2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/87624ab6620242c0a9a204034f1a0abe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/cb4d57be9f7d4394a608f0ddf1ea0af6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3fa3487f9b3b4f288a03a64532a2b975, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3a656b7bdc864291a3833ed3db117ae6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/a368f480570f4cb3a40ee2e54b23c866, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3e666d90327245e989393ff13256143a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2f5ca62c815e444382fe84d37b836664, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/ad81ea47754a4076b9ac566c656a3867, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/abbe862a5c104fa4bc7d06734d290e55, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2e2c8888af71433cb68bdcdb9009e4a3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/dfa01c973d1d4a618041d882250b25a7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/e00e0c3e045e4e7eb29a0abd33d05b24, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/a204a4e287dd4ef2b0dce656d9cb5cba, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/9291028bfa384687ac82ac3ed4a7fe94, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f959a803b47f4f9cb52979870fcea149, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/14ebebc8ada6434983a73ee0a9ab8069, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3fdf4d9232fc437faff20298e10edaff, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/ff8ad0a6b6a34ee78fe9bd5047bfe77a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c76c1c0de21c4265ad1c776bbcb53ef6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c1177776636340b2b452306215df0a7b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/576f2ba4340c43bf898cacda69670ba8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/d3c5a764ee8843af9dab7dc796a826ae, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/437d6c11094c4bb3a5fff1402d2d3023] to archive 2024-11-11T13:35:31,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:35:31,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f81b141dcb134e4e95ececba154cc883 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f81b141dcb134e4e95ececba154cc883 2024-11-11T13:35:31,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1ef5950b9f40402081d36b0fd06a5d11 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1ef5950b9f40402081d36b0fd06a5d11 2024-11-11T13:35:31,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/23c560098cb746e2ad32b1ea302af89a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/23c560098cb746e2ad32b1ea302af89a 2024-11-11T13:35:31,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/4059bb934ec94f87b8ef12fed7c9d270 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/4059bb934ec94f87b8ef12fed7c9d270 2024-11-11T13:35:31,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c07b03fbbe5b4e09ac08e530db4c326d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c07b03fbbe5b4e09ac08e530db4c326d 2024-11-11T13:35:31,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1bae6447da714f40beda8c786853ed2a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/1bae6447da714f40beda8c786853ed2a 2024-11-11T13:35:31,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/5cf5c690da274370add1892444a9ae4f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/5cf5c690da274370add1892444a9ae4f 2024-11-11T13:35:31,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/18342b3039794313bc568d208cc042fa to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/18342b3039794313bc568d208cc042fa 2024-11-11T13:35:31,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0937dc0df14947c696bbe21f458892fd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0937dc0df14947c696bbe21f458892fd 2024-11-11T13:35:31,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0d205f47a16344b794b48d26a2b2e7be to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0d205f47a16344b794b48d26a2b2e7be 2024-11-11T13:35:31,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/6d89665b683e4fc58b2a4798356ab7f2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/6d89665b683e4fc58b2a4798356ab7f2 2024-11-11T13:35:31,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/87624ab6620242c0a9a204034f1a0abe to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/87624ab6620242c0a9a204034f1a0abe 2024-11-11T13:35:31,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/cb4d57be9f7d4394a608f0ddf1ea0af6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/cb4d57be9f7d4394a608f0ddf1ea0af6 2024-11-11T13:35:31,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3fa3487f9b3b4f288a03a64532a2b975 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3fa3487f9b3b4f288a03a64532a2b975 2024-11-11T13:35:31,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3a656b7bdc864291a3833ed3db117ae6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3a656b7bdc864291a3833ed3db117ae6 2024-11-11T13:35:31,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/a368f480570f4cb3a40ee2e54b23c866 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/a368f480570f4cb3a40ee2e54b23c866 2024-11-11T13:35:31,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3e666d90327245e989393ff13256143a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3e666d90327245e989393ff13256143a 2024-11-11T13:35:31,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2f5ca62c815e444382fe84d37b836664 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2f5ca62c815e444382fe84d37b836664 2024-11-11T13:35:31,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/ad81ea47754a4076b9ac566c656a3867 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/ad81ea47754a4076b9ac566c656a3867 2024-11-11T13:35:31,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/abbe862a5c104fa4bc7d06734d290e55 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/abbe862a5c104fa4bc7d06734d290e55 2024-11-11T13:35:31,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2e2c8888af71433cb68bdcdb9009e4a3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/2e2c8888af71433cb68bdcdb9009e4a3 2024-11-11T13:35:31,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/dfa01c973d1d4a618041d882250b25a7 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/dfa01c973d1d4a618041d882250b25a7 2024-11-11T13:35:31,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/e00e0c3e045e4e7eb29a0abd33d05b24 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/e00e0c3e045e4e7eb29a0abd33d05b24 2024-11-11T13:35:31,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/a204a4e287dd4ef2b0dce656d9cb5cba to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/a204a4e287dd4ef2b0dce656d9cb5cba 2024-11-11T13:35:31,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/9291028bfa384687ac82ac3ed4a7fe94 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/9291028bfa384687ac82ac3ed4a7fe94 2024-11-11T13:35:31,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f959a803b47f4f9cb52979870fcea149 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/f959a803b47f4f9cb52979870fcea149 2024-11-11T13:35:31,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/14ebebc8ada6434983a73ee0a9ab8069 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/14ebebc8ada6434983a73ee0a9ab8069 2024-11-11T13:35:31,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3fdf4d9232fc437faff20298e10edaff to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/3fdf4d9232fc437faff20298e10edaff 2024-11-11T13:35:31,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/ff8ad0a6b6a34ee78fe9bd5047bfe77a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/ff8ad0a6b6a34ee78fe9bd5047bfe77a 2024-11-11T13:35:31,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c76c1c0de21c4265ad1c776bbcb53ef6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c76c1c0de21c4265ad1c776bbcb53ef6 2024-11-11T13:35:31,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c1177776636340b2b452306215df0a7b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/c1177776636340b2b452306215df0a7b 2024-11-11T13:35:31,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/576f2ba4340c43bf898cacda69670ba8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/576f2ba4340c43bf898cacda69670ba8 2024-11-11T13:35:31,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/d3c5a764ee8843af9dab7dc796a826ae to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/d3c5a764ee8843af9dab7dc796a826ae 2024-11-11T13:35:31,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/437d6c11094c4bb3a5fff1402d2d3023 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/437d6c11094c4bb3a5fff1402d2d3023 2024-11-11T13:35:31,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1ac8a326f10b4aeba2d02c5387c3eb48, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/21d7ccff773345359ffe0171cdafe4ad, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c4dc4263c250431c8e19032b187a3edd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/ad377a97a035475e93c43cc0fbaf6caa, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/00a067dfdf2d480189b777c40946325e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/adfbc1e598214aa2b295d4d26aa70b2d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6c162a50c4524782bc9840b19cb7de25, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6ca9b23a19a743c2ad158e5640aa2356, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/9b4ca4270873498b8f7369a5dec05a3f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/b64a5f83d04e43699245177b112484cf, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/06baa5541c7643f4a793980d1464ae36, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/14246d5ef6414357832527a65870c2e3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/dd6513afb476422297a9bf34664059e6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/5189fb7cc44f4aeca211004e7c87ab5f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/4406a0b68e8d444c95612ff0117c926b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/017714e7f2854e3cb66d12a7d31fdd26, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c3568dafd27c46fda573831a7b962938, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e6ca1080eb6840548e758350930a3e65, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/0dfb7c43ee724ab5b6cc68a5cc1731c3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/dfc497d280e049a1aa0f944389c5f955, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1eec40e3c4724184a00a7f8d645f878a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/27e0f100109b407fa290533c6c6ac5db, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/a4c631fcee3945ce8ab2174cd7911e69, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/7c1ae76795074640af38179c229abb79, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/130024a1a79440cbb02e5a8b2dd6cfc1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/bca96cbfe13a4aeebbe6bf11837d0d86, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e0368bea2abf4f77b129a3c241b75a44, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6fcbad9809ee412db9826a5686c9a615, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/fb71f96344fb4dd8997140ef15a1ee72, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/63dfbede026c45dcaa51cdf71958d052, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/59a143cae35e4a7196445938fd598699, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/be4743337b4a45d79e6995bcad049c80, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/3bcf688ff85f4d2bbc0c5154811931a1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e1555444b9564024b0728eb261c1a74c] to archive 2024-11-11T13:35:31,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:35:31,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1ac8a326f10b4aeba2d02c5387c3eb48 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1ac8a326f10b4aeba2d02c5387c3eb48 2024-11-11T13:35:31,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/21d7ccff773345359ffe0171cdafe4ad to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/21d7ccff773345359ffe0171cdafe4ad 2024-11-11T13:35:31,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c4dc4263c250431c8e19032b187a3edd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c4dc4263c250431c8e19032b187a3edd 2024-11-11T13:35:31,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/ad377a97a035475e93c43cc0fbaf6caa to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/ad377a97a035475e93c43cc0fbaf6caa 2024-11-11T13:35:31,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/00a067dfdf2d480189b777c40946325e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/00a067dfdf2d480189b777c40946325e 2024-11-11T13:35:31,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/adfbc1e598214aa2b295d4d26aa70b2d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/adfbc1e598214aa2b295d4d26aa70b2d 2024-11-11T13:35:31,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6c162a50c4524782bc9840b19cb7de25 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6c162a50c4524782bc9840b19cb7de25 2024-11-11T13:35:31,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6ca9b23a19a743c2ad158e5640aa2356 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6ca9b23a19a743c2ad158e5640aa2356 2024-11-11T13:35:31,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/9b4ca4270873498b8f7369a5dec05a3f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/9b4ca4270873498b8f7369a5dec05a3f 2024-11-11T13:35:31,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/b64a5f83d04e43699245177b112484cf to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/b64a5f83d04e43699245177b112484cf 2024-11-11T13:35:31,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/06baa5541c7643f4a793980d1464ae36 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/06baa5541c7643f4a793980d1464ae36 2024-11-11T13:35:31,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/14246d5ef6414357832527a65870c2e3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/14246d5ef6414357832527a65870c2e3 2024-11-11T13:35:31,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/dd6513afb476422297a9bf34664059e6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/dd6513afb476422297a9bf34664059e6 2024-11-11T13:35:31,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/5189fb7cc44f4aeca211004e7c87ab5f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/5189fb7cc44f4aeca211004e7c87ab5f 2024-11-11T13:35:31,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/4406a0b68e8d444c95612ff0117c926b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/4406a0b68e8d444c95612ff0117c926b 2024-11-11T13:35:31,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/017714e7f2854e3cb66d12a7d31fdd26 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/017714e7f2854e3cb66d12a7d31fdd26 2024-11-11T13:35:31,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c3568dafd27c46fda573831a7b962938 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/c3568dafd27c46fda573831a7b962938 2024-11-11T13:35:31,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e6ca1080eb6840548e758350930a3e65 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e6ca1080eb6840548e758350930a3e65 2024-11-11T13:35:31,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/0dfb7c43ee724ab5b6cc68a5cc1731c3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/0dfb7c43ee724ab5b6cc68a5cc1731c3 2024-11-11T13:35:31,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/dfc497d280e049a1aa0f944389c5f955 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/dfc497d280e049a1aa0f944389c5f955 2024-11-11T13:35:31,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1eec40e3c4724184a00a7f8d645f878a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/1eec40e3c4724184a00a7f8d645f878a 2024-11-11T13:35:31,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/27e0f100109b407fa290533c6c6ac5db to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/27e0f100109b407fa290533c6c6ac5db 2024-11-11T13:35:31,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/a4c631fcee3945ce8ab2174cd7911e69 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/a4c631fcee3945ce8ab2174cd7911e69 2024-11-11T13:35:31,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/7c1ae76795074640af38179c229abb79 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/7c1ae76795074640af38179c229abb79 2024-11-11T13:35:31,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/130024a1a79440cbb02e5a8b2dd6cfc1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/130024a1a79440cbb02e5a8b2dd6cfc1 2024-11-11T13:35:31,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/bca96cbfe13a4aeebbe6bf11837d0d86 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/bca96cbfe13a4aeebbe6bf11837d0d86 2024-11-11T13:35:31,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e0368bea2abf4f77b129a3c241b75a44 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e0368bea2abf4f77b129a3c241b75a44 2024-11-11T13:35:31,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6fcbad9809ee412db9826a5686c9a615 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/6fcbad9809ee412db9826a5686c9a615 2024-11-11T13:35:31,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/fb71f96344fb4dd8997140ef15a1ee72 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/fb71f96344fb4dd8997140ef15a1ee72 2024-11-11T13:35:31,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/63dfbede026c45dcaa51cdf71958d052 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/63dfbede026c45dcaa51cdf71958d052 2024-11-11T13:35:31,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/59a143cae35e4a7196445938fd598699 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/59a143cae35e4a7196445938fd598699 2024-11-11T13:35:31,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/be4743337b4a45d79e6995bcad049c80 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/be4743337b4a45d79e6995bcad049c80 2024-11-11T13:35:31,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/3bcf688ff85f4d2bbc0c5154811931a1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/3bcf688ff85f4d2bbc0c5154811931a1 2024-11-11T13:35:31,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e1555444b9564024b0728eb261c1a74c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/e1555444b9564024b0728eb261c1a74c 2024-11-11T13:35:31,256 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/recovered.edits/531.seqid, newMaxSeqId=531, maxSeqId=1 2024-11-11T13:35:31,257 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. 2024-11-11T13:35:31,257 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 3b71678035107ffe2dbde3bb5bfff436: Waiting for close lock at 1731332129172Running coprocessor pre-close hooks at 1731332129172Disabling compacts and flushes for region at 1731332129172Disabling writes for close at 1731332129792 (+620 ms)Obtaining lock to block concurrent updates at 1731332129792Preparing flush snapshotting stores in 3b71678035107ffe2dbde3bb5bfff436 at 1731332129792Finished memstore snapshotting TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731332129792Flushing stores of TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436. at 1731332129793 (+1 ms)Flushing 3b71678035107ffe2dbde3bb5bfff436/A: creating writer at 1731332129793Flushing 3b71678035107ffe2dbde3bb5bfff436/A: appending metadata at 1731332129795 (+2 ms)Flushing 3b71678035107ffe2dbde3bb5bfff436/A: closing flushed file at 1731332129795Flushing 3b71678035107ffe2dbde3bb5bfff436/B: creating writer at 1731332130212 (+417 ms)Flushing 3b71678035107ffe2dbde3bb5bfff436/B: appending metadata at 1731332130215 (+3 ms)Flushing 3b71678035107ffe2dbde3bb5bfff436/B: closing flushed file at 1731332130215Flushing 3b71678035107ffe2dbde3bb5bfff436/C: creating writer at 1731332130652 (+437 ms)Flushing 3b71678035107ffe2dbde3bb5bfff436/C: appending metadata at 1731332130656 (+4 ms)Flushing 3b71678035107ffe2dbde3bb5bfff436/C: closing flushed file at 1731332130657 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7881e89e: reopening flushed file at 1731332131071 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b5a5bdf: reopening flushed file at 1731332131078 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3939dd2b: reopening flushed file at 1731332131085 (+7 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 3b71678035107ffe2dbde3bb5bfff436 in 1300ms, sequenceid=528, compaction requested=true at 1731332131092 (+7 ms)Writing region close event to WAL at 1731332131252 (+160 ms)Running coprocessor post-close hooks at 1731332131257 (+5 ms)Closed at 1731332131257 2024-11-11T13:35:31,259 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:31,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=3b71678035107ffe2dbde3bb5bfff436, regionState=CLOSED 2024-11-11T13:35:31,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:35:31,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T13:35:31,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 3b71678035107ffe2dbde3bb5bfff436, server=37d26ecf6460,36877,1731332101576 in 2.2510 sec 2024-11-11T13:35:31,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-11T13:35:31,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3b71678035107ffe2dbde3bb5bfff436, UNASSIGN in 2.2590 sec 2024-11-11T13:35:31,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T13:35:31,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2650 sec 2024-11-11T13:35:31,270 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332131270"}]},"ts":"1731332131270"} 2024-11-11T13:35:31,272 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T13:35:31,272 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T13:35:31,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2780 sec 2024-11-11T13:35:33,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T13:35:33,171 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:33,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T13:35:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:35:33,187 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:35:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T13:35:33,189 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:35:33,193 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:33,197 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/recovered.edits] 2024-11-11T13:35:33,200 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/149e4960779a44c493c2f90b664570dc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/149e4960779a44c493c2f90b664570dc 2024-11-11T13:35:33,202 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9e0696ac548845788c42f9798ccd8f28 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/9e0696ac548845788c42f9798ccd8f28 2024-11-11T13:35:33,203 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a9067cf2e3c346ad8f11ebacaf80a04c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/a9067cf2e3c346ad8f11ebacaf80a04c 2024-11-11T13:35:33,205 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/cc4745a9ae91493f90c9025ec60b062c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/A/cc4745a9ae91493f90c9025ec60b062c 2024-11-11T13:35:33,208 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/043ef00548f44a94b16ba045a86de5a7 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/043ef00548f44a94b16ba045a86de5a7 2024-11-11T13:35:33,209 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0c6cebc249b944838d10ed5e836a45e6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/0c6cebc249b944838d10ed5e836a45e6 2024-11-11T13:35:33,211 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/4a0f323a28e74f9aaf3b84daaa9e38fa to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/4a0f323a28e74f9aaf3b84daaa9e38fa 2024-11-11T13:35:33,212 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/622f4e80096f4e72be6436ffda98398c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/B/622f4e80096f4e72be6436ffda98398c 2024-11-11T13:35:33,214 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/304c7d64a22e49f08c5acf85dbd5ffe2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/304c7d64a22e49f08c5acf85dbd5ffe2 2024-11-11T13:35:33,215 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/3e6eecaa33f5442f97b8816d76963a52 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/3e6eecaa33f5442f97b8816d76963a52 2024-11-11T13:35:33,216 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/736118cff3f44aa18a5566a481300c37 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/736118cff3f44aa18a5566a481300c37 2024-11-11T13:35:33,217 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/a59c13a6d7cf4ff98068d3bda2526463 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/C/a59c13a6d7cf4ff98068d3bda2526463 2024-11-11T13:35:33,219 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/recovered.edits/531.seqid to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436/recovered.edits/531.seqid 2024-11-11T13:35:33,220 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/3b71678035107ffe2dbde3bb5bfff436 2024-11-11T13:35:33,220 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T13:35:33,225 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:35:33,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T13:35:33,233 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T13:35:33,258 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T13:35:33,260 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:35:33,260 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T13:35:33,260 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731332133260"}]},"ts":"9223372036854775807"} 2024-11-11T13:35:33,264 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T13:35:33,264 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3b71678035107ffe2dbde3bb5bfff436, NAME => 'TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T13:35:33,265 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T13:35:33,265 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731332133265"}]},"ts":"9223372036854775807"} 2024-11-11T13:35:33,268 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T13:35:33,268 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:35:33,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 87 msec 2024-11-11T13:35:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T13:35:33,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T13:35:33,300 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:33,313 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_567230531_22 at /127.0.0.1:44370 [Waiting for operation #127] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_567230531_22 at /127.0.0.1:49026 [Waiting for operation #284] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;37d26ecf6460:36877-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-931856926_22 at /127.0.0.1:54796 [Waiting for operation #307] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=490 (was 330) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3099 (was 3489) 2024-11-11T13:35:33,322 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=490, ProcessCount=11, AvailableMemoryMB=3099 2024-11-11T13:35:33,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T13:35:33,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:35:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T13:35:33,327 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T13:35:33,327 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:33,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-11T13:35:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T13:35:33,329 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T13:35:33,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741951_1127 (size=959) 2024-11-11T13:35:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T13:35:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T13:35:33,739 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92 2024-11-11T13:35:33,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741952_1128 (size=53) 2024-11-11T13:35:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:35:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e913da51831f130352321c03b81d22d5, disabling compactions & flushes 2024-11-11T13:35:33,750 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. after waiting 0 ms 2024-11-11T13:35:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:33,750 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:33,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e913da51831f130352321c03b81d22d5: Waiting for close lock at 1731332133750Disabling compacts and flushes for region at 1731332133750Disabling writes for close at 1731332133750Writing region close event to WAL at 1731332133750Closed at 1731332133750 2024-11-11T13:35:33,752 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T13:35:33,752 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731332133752"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731332133752"}]},"ts":"1731332133752"} 2024-11-11T13:35:33,754 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T13:35:33,755 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T13:35:33,756 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332133755"}]},"ts":"1731332133755"} 2024-11-11T13:35:33,758 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T13:35:33,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, ASSIGN}] 2024-11-11T13:35:33,760 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, ASSIGN 2024-11-11T13:35:33,761 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, ASSIGN; state=OFFLINE, location=37d26ecf6460,36877,1731332101576; forceNewPlan=false, retain=false 2024-11-11T13:35:33,912 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=e913da51831f130352321c03b81d22d5, regionState=OPENING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:33,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, ASSIGN because future has completed 2024-11-11T13:35:33,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:35:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T13:35:34,073 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:34,073 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:35:34,074 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,074 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:35:34,074 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,074 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,076 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,079 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e913da51831f130352321c03b81d22d5 columnFamilyName A 2024-11-11T13:35:34,079 DEBUG [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:34,080 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(327): Store=e913da51831f130352321c03b81d22d5/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:34,080 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,081 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e913da51831f130352321c03b81d22d5 columnFamilyName B 2024-11-11T13:35:34,081 DEBUG [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:34,081 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(327): Store=e913da51831f130352321c03b81d22d5/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:34,081 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,082 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e913da51831f130352321c03b81d22d5 columnFamilyName C 2024-11-11T13:35:34,082 DEBUG [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:34,083 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(327): Store=e913da51831f130352321c03b81d22d5/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:34,083 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,083 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,084 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,085 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,085 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,085 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T13:35:34,086 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,087 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:35:34,088 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened e913da51831f130352321c03b81d22d5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72135297, jitterRate=0.07489968836307526}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T13:35:34,088 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:34,089 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for e913da51831f130352321c03b81d22d5: Running coprocessor pre-open hook at 1731332134074Writing region info on filesystem at 1731332134074Initializing all the Stores at 1731332134075 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332134076 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332134076Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332134076Cleaning up temporary data from old regions at 1731332134085 (+9 ms)Running coprocessor post-open hooks at 1731332134088 (+3 ms)Region opened successfully at 1731332134089 (+1 ms) 2024-11-11T13:35:34,089 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., pid=32, masterSystemTime=1731332134069 2024-11-11T13:35:34,091 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:34,091 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:34,092 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=e913da51831f130352321c03b81d22d5, regionState=OPEN, openSeqNum=2, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:34,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:35:34,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-11T13:35:34,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 in 179 msec 2024-11-11T13:35:34,098 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-11T13:35:34,098 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, ASSIGN in 338 msec 2024-11-11T13:35:34,099 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T13:35:34,099 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332134099"}]},"ts":"1731332134099"} 2024-11-11T13:35:34,101 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T13:35:34,102 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T13:35:34,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 777 msec 2024-11-11T13:35:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-11T13:35:34,470 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:34,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae37b4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:34,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:34,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:34,472 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcec947, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:34,474 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:34,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93c960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:34,475 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:34,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:34,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:34,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:34,481 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T13:35:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 37d26ecf6460,35609,1731332100707 2024-11-11T13:35:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2022ca19 2024-11-11T13:35:34,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T13:35:34,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T13:35:34,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T13:35:34,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:35:34,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T13:35:34,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741953_1129 (size=995) 2024-11-11T13:35:34,914 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T13:35:34,914 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T13:35:34,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T13:35:34,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, REOPEN/MOVE}] 2024-11-11T13:35:34,924 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, REOPEN/MOVE 2024-11-11T13:35:34,925 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e913da51831f130352321c03b81d22d5, regionState=CLOSING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:34,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, REOPEN/MOVE because future has completed 2024-11-11T13:35:34,927 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T13:35:34,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:35:35,080 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,080 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T13:35:35,080 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing e913da51831f130352321c03b81d22d5, disabling compactions & flushes 2024-11-11T13:35:35,080 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,080 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,080 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. after waiting 0 ms 2024-11-11T13:35:35,080 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,086 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T13:35:35,087 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,087 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for e913da51831f130352321c03b81d22d5: Waiting for close lock at 1731332135080Running coprocessor pre-close hooks at 1731332135080Disabling compacts and flushes for region at 1731332135080Disabling writes for close at 1731332135080Writing region close event to WAL at 1731332135081 (+1 ms)Running coprocessor post-close hooks at 1731332135087 (+6 ms)Closed at 1731332135087 2024-11-11T13:35:35,087 WARN [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: e913da51831f130352321c03b81d22d5 to self. 2024-11-11T13:35:35,089 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,090 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e913da51831f130352321c03b81d22d5, regionState=CLOSED 2024-11-11T13:35:35,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:35:35,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-11T13:35:35,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 in 166 msec 2024-11-11T13:35:35,095 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, REOPEN/MOVE; state=CLOSED, location=37d26ecf6460,36877,1731332101576; forceNewPlan=false, retain=true 2024-11-11T13:35:35,246 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e913da51831f130352321c03b81d22d5, regionState=OPENING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, REOPEN/MOVE because future has completed 2024-11-11T13:35:35,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:35:35,404 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,404 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:35:35,405 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,405 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:35:35,405 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,405 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,409 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,414 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e913da51831f130352321c03b81d22d5 columnFamilyName A 2024-11-11T13:35:35,417 DEBUG [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:35,417 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(327): Store=e913da51831f130352321c03b81d22d5/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:35,418 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,418 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e913da51831f130352321c03b81d22d5 columnFamilyName B 2024-11-11T13:35:35,418 DEBUG [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:35,419 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(327): Store=e913da51831f130352321c03b81d22d5/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:35,419 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,420 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e913da51831f130352321c03b81d22d5 columnFamilyName C 2024-11-11T13:35:35,420 DEBUG [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:35,421 INFO [StoreOpener-e913da51831f130352321c03b81d22d5-1 {}] regionserver.HStore(327): Store=e913da51831f130352321c03b81d22d5/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:35:35,421 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,422 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,423 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,424 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,424 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,425 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T13:35:35,427 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,428 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened e913da51831f130352321c03b81d22d5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62624043, jitterRate=-0.06682904064655304}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T13:35:35,428 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,429 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for e913da51831f130352321c03b81d22d5: Running coprocessor pre-open hook at 1731332135405Writing region info on filesystem at 1731332135405Initializing all the Stores at 1731332135406 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332135406Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332135406Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332135406Cleaning up temporary data from old regions at 1731332135424 (+18 ms)Running coprocessor post-open hooks at 1731332135428 (+4 ms)Region opened successfully at 1731332135429 (+1 ms) 2024-11-11T13:35:35,430 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., pid=37, masterSystemTime=1731332135401 2024-11-11T13:35:35,432 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,432 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,433 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e913da51831f130352321c03b81d22d5, regionState=OPEN, openSeqNum=5, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:35:35,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-11T13:35:35,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 in 187 msec 2024-11-11T13:35:35,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-11T13:35:35,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, REOPEN/MOVE in 515 msec 2024-11-11T13:35:35,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-11T13:35:35,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 524 msec 2024-11-11T13:35:35,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 954 msec 2024-11-11T13:35:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T13:35:35,448 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:35,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b2d6ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:35,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:35,450 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a78cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:35,451 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ef18fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2047fc89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:35,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:35,454 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb02826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:35,455 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:35,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25cc23f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4afde3a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,457 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:35,457 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:35,458 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f32e13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:35,459 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33ef71ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f288857, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,461 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:35,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:35,462 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b27bd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:35,463 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7369fec8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d9552a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:35,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:35,466 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcf6991, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:35,467 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1370feea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@437c4c13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:35,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:35,471 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b337843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:35,473 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2106acc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dace243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,475 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:35,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:35,476 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:35,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:35,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68926b50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:35,478 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:35,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd05b83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8714dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:35,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:35,481 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ad41be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:35,483 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:35,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4abed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6986f6f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:35:35,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:35:35,486 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:35:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:35:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:35:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9e4beb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:35:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:35:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:35,488 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:35:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce03a8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:35:35,490 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:35,491 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:35,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:35,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:35,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:35,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:35:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:35:35,502 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:35,502 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:35,504 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:35,504 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:35,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:35,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:35,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:35,507 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:35,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-11T13:35:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:35:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T13:35:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:35:35,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:35,512 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:35,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:35:35,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:35:35,516 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:35,517 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:35,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:35,518 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:35:35,519 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:35,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:35,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:35:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:35:35,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:35,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:35:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:35:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:35:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:35:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:35,529 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:35,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:35:35,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332195598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332195599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332195603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T13:35:35,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332195610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332195611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111179d61fe79a054657af5b0072a9b38a00_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332135517/Put/seqid=0 2024-11-11T13:35:35,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T13:35:35,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:35,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:35,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:35,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741954_1130 (size=17034) 2024-11-11T13:35:35,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332195723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332195723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332195724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332195731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332195744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T13:35:35,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T13:35:35,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:35,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:35,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:35,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332195940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332195940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332195940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332195950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332195959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:35,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:35,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T13:35:35,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:35,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:35,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:35,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:36,084 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,091 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111179d61fe79a054657af5b0072a9b38a00_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111179d61fe79a054657af5b0072a9b38a00_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:36,093 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/80abe52aa2fa4eb2b3ac841278709ffe, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:36,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/80abe52aa2fa4eb2b3ac841278709ffe is 175, key is test_row_0/A:col10/1731332135517/Put/seqid=0 2024-11-11T13:35:36,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741955_1131 (size=48139) 2024-11-11T13:35:36,114 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/80abe52aa2fa4eb2b3ac841278709ffe 2024-11-11T13:35:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T13:35:36,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T13:35:36,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:36,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:36,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:36,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:36,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:36,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/18a2cd82f0b843efbd1b09317e25bae4 is 50, key is test_row_0/B:col10/1731332135517/Put/seqid=0 2024-11-11T13:35:36,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741956_1132 (size=12001) 2024-11-11T13:35:36,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/18a2cd82f0b843efbd1b09317e25bae4 2024-11-11T13:35:36,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/5e0e86423c27462a8089672d26e3137a is 50, key is test_row_0/C:col10/1731332135517/Put/seqid=0 2024-11-11T13:35:36,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741957_1133 (size=12001) 2024-11-11T13:35:36,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/5e0e86423c27462a8089672d26e3137a 2024-11-11T13:35:36,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/80abe52aa2fa4eb2b3ac841278709ffe as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/80abe52aa2fa4eb2b3ac841278709ffe 2024-11-11T13:35:36,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/80abe52aa2fa4eb2b3ac841278709ffe, entries=250, sequenceid=15, filesize=47.0 K 2024-11-11T13:35:36,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/18a2cd82f0b843efbd1b09317e25bae4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/18a2cd82f0b843efbd1b09317e25bae4 2024-11-11T13:35:36,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332196250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:36,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:36,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:36,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:36,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332196260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:36,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:36,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332196260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332196260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:36,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:36,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332196271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:36,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:36,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:36,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:36,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/18a2cd82f0b843efbd1b09317e25bae4, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T13:35:36,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/5e0e86423c27462a8089672d26e3137a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/5e0e86423c27462a8089672d26e3137a 2024-11-11T13:35:36,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/5e0e86423c27462a8089672d26e3137a, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T13:35:36,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e913da51831f130352321c03b81d22d5 in 764ms, sequenceid=15, compaction requested=false 2024-11-11T13:35:36,293 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T13:35:36,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T13:35:36,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:36,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:36,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:36,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:36,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:36,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:36,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T13:35:36,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:36,455 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T13:35:36,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111427d66d5deda4750bc409a06ff3fc51f_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332135607/Put/seqid=0 2024-11-11T13:35:36,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741958_1134 (size=12154) 2024-11-11T13:35:36,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,531 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111427d66d5deda4750bc409a06ff3fc51f_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111427d66d5deda4750bc409a06ff3fc51f_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:36,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a2384366e23e4262bd29dbebdf841efd, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:36,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a2384366e23e4262bd29dbebdf841efd is 175, key is test_row_0/A:col10/1731332135607/Put/seqid=0 2024-11-11T13:35:36,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741959_1135 (size=30955) 2024-11-11T13:35:36,584 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a2384366e23e4262bd29dbebdf841efd 2024-11-11T13:35:36,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/a1343df06f8e4a578405cce9347dd7c5 is 50, key is test_row_0/B:col10/1731332135607/Put/seqid=0 2024-11-11T13:35:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T13:35:36,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741960_1136 (size=12001) 2024-11-11T13:35:36,664 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/a1343df06f8e4a578405cce9347dd7c5 2024-11-11T13:35:36,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/86f0867589bf4104ac8348ed0f4aa3db is 50, key is test_row_0/C:col10/1731332135607/Put/seqid=0 2024-11-11T13:35:36,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741961_1137 (size=12001) 2024-11-11T13:35:36,711 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/86f0867589bf4104ac8348ed0f4aa3db 2024-11-11T13:35:36,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a2384366e23e4262bd29dbebdf841efd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a2384366e23e4262bd29dbebdf841efd 2024-11-11T13:35:36,774 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a2384366e23e4262bd29dbebdf841efd, entries=150, sequenceid=41, filesize=30.2 K 2024-11-11T13:35:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/a1343df06f8e4a578405cce9347dd7c5 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/a1343df06f8e4a578405cce9347dd7c5 2024-11-11T13:35:36,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,785 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/a1343df06f8e4a578405cce9347dd7c5, entries=150, sequenceid=41, filesize=11.7 K 2024-11-11T13:35:36,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/86f0867589bf4104ac8348ed0f4aa3db as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/86f0867589bf4104ac8348ed0f4aa3db 2024-11-11T13:35:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,794 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/86f0867589bf4104ac8348ed0f4aa3db, entries=150, sequenceid=41, filesize=11.7 K 2024-11-11T13:35:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,795 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=20.13 KB/20610 for e913da51831f130352321c03b81d22d5 in 341ms, sequenceid=41, compaction requested=false 2024-11-11T13:35:36,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:36,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:36,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-11T13:35:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-11T13:35:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-11T13:35:36,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2780 sec 2024-11-11T13:35:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.2970 sec 2024-11-11T13:35:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,811 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T13:35:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:36,899 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:36,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:35:36,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111120fd7efde95b49178565560a84a84506_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332136792/Put/seqid=0 2024-11-11T13:35:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:36,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741963_1139 (size=24358) 2024-11-11T13:35:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332197011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332197018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332197021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332197023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332197026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332197140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332197140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332197140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332197140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332197141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332197360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332197360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332197361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332197361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332197362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,408 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:37,413 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111120fd7efde95b49178565560a84a84506_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111120fd7efde95b49178565560a84a84506_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:37,415 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/c8f3cc36735947e1a304cb6b59d42fe4, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:37,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/c8f3cc36735947e1a304cb6b59d42fe4 is 175, key is test_row_0/A:col10/1731332136792/Put/seqid=0 2024-11-11T13:35:37,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741962_1138 (size=73995) 2024-11-11T13:35:37,420 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/c8f3cc36735947e1a304cb6b59d42fe4 2024-11-11T13:35:37,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/b5d253a368d945a9ab6e73c13ece505b is 50, key is test_row_0/B:col10/1731332136792/Put/seqid=0 2024-11-11T13:35:37,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741964_1140 (size=12001) 2024-11-11T13:35:37,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/b5d253a368d945a9ab6e73c13ece505b 2024-11-11T13:35:37,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/250e367a70ee494793141298da4b0d70 is 50, key is test_row_0/C:col10/1731332136792/Put/seqid=0 2024-11-11T13:35:37,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741965_1141 (size=12001) 2024-11-11T13:35:37,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/250e367a70ee494793141298da4b0d70 2024-11-11T13:35:37,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/c8f3cc36735947e1a304cb6b59d42fe4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c8f3cc36735947e1a304cb6b59d42fe4 2024-11-11T13:35:37,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c8f3cc36735947e1a304cb6b59d42fe4, entries=400, sequenceid=52, filesize=72.3 K 2024-11-11T13:35:37,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/b5d253a368d945a9ab6e73c13ece505b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b5d253a368d945a9ab6e73c13ece505b 2024-11-11T13:35:37,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b5d253a368d945a9ab6e73c13ece505b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T13:35:37,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/250e367a70ee494793141298da4b0d70 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/250e367a70ee494793141298da4b0d70 2024-11-11T13:35:37,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/250e367a70ee494793141298da4b0d70, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T13:35:37,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e913da51831f130352321c03b81d22d5 in 629ms, sequenceid=52, compaction requested=true 2024-11-11T13:35:37,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:37,529 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:37,530 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 153089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:37,530 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/A is initiating minor compaction (all files) 2024-11-11T13:35:37,530 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/A in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:37,530 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/80abe52aa2fa4eb2b3ac841278709ffe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a2384366e23e4262bd29dbebdf841efd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c8f3cc36735947e1a304cb6b59d42fe4] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=149.5 K 2024-11-11T13:35:37,530 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:37,531 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/80abe52aa2fa4eb2b3ac841278709ffe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a2384366e23e4262bd29dbebdf841efd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c8f3cc36735947e1a304cb6b59d42fe4] 2024-11-11T13:35:37,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:37,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:37,531 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:37,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:37,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:37,531 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80abe52aa2fa4eb2b3ac841278709ffe, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731332135511 2024-11-11T13:35:37,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:37,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:37,532 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2384366e23e4262bd29dbebdf841efd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731332135579 2024-11-11T13:35:37,532 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:37,532 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/B is initiating minor compaction (all files) 2024-11-11T13:35:37,532 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/B in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:37,532 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/18a2cd82f0b843efbd1b09317e25bae4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/a1343df06f8e4a578405cce9347dd7c5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b5d253a368d945a9ab6e73c13ece505b] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=35.2 K 2024-11-11T13:35:37,533 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8f3cc36735947e1a304cb6b59d42fe4, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332136774 2024-11-11T13:35:37,533 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a2cd82f0b843efbd1b09317e25bae4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731332135511 2024-11-11T13:35:37,533 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting a1343df06f8e4a578405cce9347dd7c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731332135579 2024-11-11T13:35:37,534 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d253a368d945a9ab6e73c13ece505b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332136792 2024-11-11T13:35:37,552 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:37,559 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:37,559 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/2bd528faf1f747c1a12ad9a52a2a8c85 is 50, key is test_row_0/B:col10/1731332136792/Put/seqid=0 2024-11-11T13:35:37,566 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118065e74aa97b45258852cb82611ab604_e913da51831f130352321c03b81d22d5 store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:37,571 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118065e74aa97b45258852cb82611ab604_e913da51831f130352321c03b81d22d5, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:37,571 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118065e74aa97b45258852cb82611ab604_e913da51831f130352321c03b81d22d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:37,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741966_1142 (size=12104) 2024-11-11T13:35:37,600 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/2bd528faf1f747c1a12ad9a52a2a8c85 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/2bd528faf1f747c1a12ad9a52a2a8c85 2024-11-11T13:35:37,608 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e913da51831f130352321c03b81d22d5/B of e913da51831f130352321c03b81d22d5 into 2bd528faf1f747c1a12ad9a52a2a8c85(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:37,608 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:37,608 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/B, priority=13, startTime=1731332137531; duration=0sec 2024-11-11T13:35:37,609 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:37,609 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:B 2024-11-11T13:35:37,609 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:37,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741967_1143 (size=4469) 2024-11-11T13:35:37,611 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:37,612 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/C is initiating minor compaction (all files) 2024-11-11T13:35:37,612 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/C in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:37,612 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/5e0e86423c27462a8089672d26e3137a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/86f0867589bf4104ac8348ed0f4aa3db, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/250e367a70ee494793141298da4b0d70] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=35.2 K 2024-11-11T13:35:37,613 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e0e86423c27462a8089672d26e3137a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731332135511 2024-11-11T13:35:37,614 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f0867589bf4104ac8348ed0f4aa3db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731332135579 2024-11-11T13:35:37,614 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#A#compaction#123 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:37,615 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/245c2792e1944ca6a0999123c742d624 is 175, key is test_row_0/A:col10/1731332136792/Put/seqid=0 2024-11-11T13:35:37,615 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 250e367a70ee494793141298da4b0d70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332136792 2024-11-11T13:35:37,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741968_1144 (size=31058) 2024-11-11T13:35:37,637 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:37,637 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/04980b51d3d94715bbfb174a1c4ddbcd is 50, key is test_row_0/C:col10/1731332136792/Put/seqid=0 2024-11-11T13:35:37,646 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/245c2792e1944ca6a0999123c742d624 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/245c2792e1944ca6a0999123c742d624 2024-11-11T13:35:37,655 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e913da51831f130352321c03b81d22d5/A of e913da51831f130352321c03b81d22d5 into 245c2792e1944ca6a0999123c742d624(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:37,655 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:37,655 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/A, priority=13, startTime=1731332137529; duration=0sec 2024-11-11T13:35:37,655 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:37,655 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:A 2024-11-11T13:35:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T13:35:37,662 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:37,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:37,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741969_1145 (size=12104) 2024-11-11T13:35:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-11T13:35:37,674 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:37,675 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:37,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:37,676 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/04980b51d3d94715bbfb174a1c4ddbcd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/04980b51d3d94715bbfb174a1c4ddbcd 2024-11-11T13:35:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T13:35:37,678 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:37,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T13:35:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:37,690 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e913da51831f130352321c03b81d22d5/C of e913da51831f130352321c03b81d22d5 into 04980b51d3d94715bbfb174a1c4ddbcd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:37,690 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:37,690 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/C, priority=13, startTime=1731332137531; duration=0sec 2024-11-11T13:35:37,690 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:37,690 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:C 2024-11-11T13:35:37,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332197690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332197687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332197692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332197701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332197701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111096ad2fdd5054e76a105c4e0ec8efca6_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332137672/Put/seqid=0 2024-11-11T13:35:37,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741970_1146 (size=17034) 2024-11-11T13:35:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T13:35:37,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332197810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332197811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332197811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332197811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332197811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:37,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:37,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:37,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:37,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:37,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:37,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:37,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:37,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T13:35:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332198019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332198020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332198020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332198020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332198021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:38,159 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111096ad2fdd5054e76a105c4e0ec8efca6_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111096ad2fdd5054e76a105c4e0ec8efca6_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:38,161 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/495fd2ab746b448abbfb64b606863a6f, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:38,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/495fd2ab746b448abbfb64b606863a6f is 175, key is test_row_0/A:col10/1731332137672/Put/seqid=0 2024-11-11T13:35:38,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741971_1147 (size=48139) 2024-11-11T13:35:38,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T13:35:38,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332198330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332198330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332198331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332198331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332198331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,567 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/495fd2ab746b448abbfb64b606863a6f 2024-11-11T13:35:38,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/1e1ebaf4c4f942498d571537dc6cafc7 is 50, key is test_row_0/B:col10/1731332137672/Put/seqid=0 2024-11-11T13:35:38,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741972_1148 (size=12001) 2024-11-11T13:35:38,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:38,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:38,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:38,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T13:35:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332198849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332198849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332198850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332198850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332198850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:38,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:38,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:39,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/1e1ebaf4c4f942498d571537dc6cafc7 2024-11-11T13:35:39,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/787f597937564fadbbc0b148676e4430 is 50, key is test_row_0/C:col10/1731332137672/Put/seqid=0 2024-11-11T13:35:39,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741973_1149 (size=12001) 2024-11-11T13:35:39,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:39,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:39,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:39,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:39,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:39,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:39,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:39,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:39,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:39,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:39,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:39,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:39,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:39,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:39,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:39,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:39,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:39,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:39,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/787f597937564fadbbc0b148676e4430 2024-11-11T13:35:39,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/495fd2ab746b448abbfb64b606863a6f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/495fd2ab746b448abbfb64b606863a6f 2024-11-11T13:35:39,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/495fd2ab746b448abbfb64b606863a6f, entries=250, sequenceid=81, filesize=47.0 K 2024-11-11T13:35:39,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/1e1ebaf4c4f942498d571537dc6cafc7 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1e1ebaf4c4f942498d571537dc6cafc7 2024-11-11T13:35:39,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1e1ebaf4c4f942498d571537dc6cafc7, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T13:35:39,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/787f597937564fadbbc0b148676e4430 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/787f597937564fadbbc0b148676e4430 2024-11-11T13:35:39,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/787f597937564fadbbc0b148676e4430, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T13:35:39,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for e913da51831f130352321c03b81d22d5 in 1827ms, sequenceid=81, compaction requested=false 2024-11-11T13:35:39,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:39,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T13:35:39,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:39,555 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-11T13:35:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110cbdfccbe6fb4f518ef316688b7d77ec_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332137689/Put/seqid=0 2024-11-11T13:35:39,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741974_1150 (size=12154) 2024-11-11T13:35:39,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:39,647 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110cbdfccbe6fb4f518ef316688b7d77ec_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110cbdfccbe6fb4f518ef316688b7d77ec_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:39,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/8c0ad0f4f68e4891b0964b0c28e4ea08, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:39,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/8c0ad0f4f68e4891b0964b0c28e4ea08 is 175, key is test_row_0/A:col10/1731332137689/Put/seqid=0 2024-11-11T13:35:39,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741975_1151 (size=30955) 2024-11-11T13:35:39,687 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/8c0ad0f4f68e4891b0964b0c28e4ea08 2024-11-11T13:35:39,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/c567fe0b1da3469f90e9df9fabc78c20 is 50, key is test_row_0/B:col10/1731332137689/Put/seqid=0 2024-11-11T13:35:39,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741976_1152 (size=12001) 2024-11-11T13:35:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T13:35:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:39,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:39,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332199894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:39,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:39,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:39,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332199896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332199896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:39,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332199897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:39,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332199899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332200009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332200010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332200011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332200011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332200010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,128 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/c567fe0b1da3469f90e9df9fabc78c20 2024-11-11T13:35:40,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/d3b4ca82302248e5ba6cfaf3eb6a184b is 50, key is test_row_0/C:col10/1731332137689/Put/seqid=0 2024-11-11T13:35:40,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741977_1153 (size=12001) 2024-11-11T13:35:40,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332200230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332200231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332200230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332200231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332200231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332200539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332200550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332200550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332200550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332200550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:40,596 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/d3b4ca82302248e5ba6cfaf3eb6a184b 2024-11-11T13:35:40,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/8c0ad0f4f68e4891b0964b0c28e4ea08 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/8c0ad0f4f68e4891b0964b0c28e4ea08 2024-11-11T13:35:40,627 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/8c0ad0f4f68e4891b0964b0c28e4ea08, entries=150, sequenceid=92, filesize=30.2 K 2024-11-11T13:35:40,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/c567fe0b1da3469f90e9df9fabc78c20 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/c567fe0b1da3469f90e9df9fabc78c20 2024-11-11T13:35:40,657 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/c567fe0b1da3469f90e9df9fabc78c20, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T13:35:40,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/d3b4ca82302248e5ba6cfaf3eb6a184b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b4ca82302248e5ba6cfaf3eb6a184b 2024-11-11T13:35:40,668 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b4ca82302248e5ba6cfaf3eb6a184b, entries=150, sequenceid=92, filesize=11.7 K 2024-11-11T13:35:40,672 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e913da51831f130352321c03b81d22d5 in 1117ms, sequenceid=92, compaction requested=true 2024-11-11T13:35:40,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:40,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:40,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-11T13:35:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-11T13:35:40,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-11T13:35:40,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0000 sec 2024-11-11T13:35:40,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.0160 sec 2024-11-11T13:35:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:41,064 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:41,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T13:35:41,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332201074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332201078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332201081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332201083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fa69bce6e98b4a738c325fc1b96cc9b2_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332139896/Put/seqid=0 2024-11-11T13:35:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332201091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741978_1154 (size=12154) 2024-11-11T13:35:41,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332201202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332201203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332201203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332201210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332201211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332201423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332201423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332201423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332201424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332201423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,579 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:41,609 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fa69bce6e98b4a738c325fc1b96cc9b2_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fa69bce6e98b4a738c325fc1b96cc9b2_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:41,610 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/272f0a868c9c463a80c4269842243755, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:41,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/272f0a868c9c463a80c4269842243755 is 175, key is test_row_0/A:col10/1731332139896/Put/seqid=0 2024-11-11T13:35:41,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741979_1155 (size=30955) 2024-11-11T13:35:41,627 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/272f0a868c9c463a80c4269842243755 2024-11-11T13:35:41,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/20b3c772763046e3b420520fc59e264c is 50, key is test_row_0/B:col10/1731332139896/Put/seqid=0 2024-11-11T13:35:41,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741980_1156 (size=12001) 2024-11-11T13:35:41,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/20b3c772763046e3b420520fc59e264c 2024-11-11T13:35:41,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/bfaff851454d428e932486af0b7dcf02 is 50, key is test_row_0/C:col10/1731332139896/Put/seqid=0 2024-11-11T13:35:41,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741981_1157 (size=12001) 2024-11-11T13:35:41,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/bfaff851454d428e932486af0b7dcf02 2024-11-11T13:35:41,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/272f0a868c9c463a80c4269842243755 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/272f0a868c9c463a80c4269842243755 2024-11-11T13:35:41,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/272f0a868c9c463a80c4269842243755, entries=150, sequenceid=120, filesize=30.2 K 2024-11-11T13:35:41,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/20b3c772763046e3b420520fc59e264c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/20b3c772763046e3b420520fc59e264c 2024-11-11T13:35:41,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/20b3c772763046e3b420520fc59e264c, entries=150, sequenceid=120, filesize=11.7 K 2024-11-11T13:35:41,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/bfaff851454d428e932486af0b7dcf02 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/bfaff851454d428e932486af0b7dcf02 2024-11-11T13:35:41,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/bfaff851454d428e932486af0b7dcf02, entries=150, sequenceid=120, filesize=11.7 K 2024-11-11T13:35:41,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for e913da51831f130352321c03b81d22d5 in 673ms, sequenceid=120, compaction requested=true 2024-11-11T13:35:41,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:41,737 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:41,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:41,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:41,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:41,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:41,738 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:41,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:41,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:41,739 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:41,739 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:41,739 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/A is initiating minor compaction (all files) 2024-11-11T13:35:41,739 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/B is initiating minor compaction (all files) 2024-11-11T13:35:41,739 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/A in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:41,739 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/245c2792e1944ca6a0999123c742d624, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/495fd2ab746b448abbfb64b606863a6f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/8c0ad0f4f68e4891b0964b0c28e4ea08, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/272f0a868c9c463a80c4269842243755] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=137.8 K 2024-11-11T13:35:41,739 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:41,739 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/245c2792e1944ca6a0999123c742d624, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/495fd2ab746b448abbfb64b606863a6f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/8c0ad0f4f68e4891b0964b0c28e4ea08, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/272f0a868c9c463a80c4269842243755] 2024-11-11T13:35:41,740 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/B in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:41,741 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/2bd528faf1f747c1a12ad9a52a2a8c85, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1e1ebaf4c4f942498d571537dc6cafc7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/c567fe0b1da3469f90e9df9fabc78c20, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/20b3c772763046e3b420520fc59e264c] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=47.0 K 2024-11-11T13:35:41,741 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 245c2792e1944ca6a0999123c742d624, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332136792 2024-11-11T13:35:41,742 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bd528faf1f747c1a12ad9a52a2a8c85, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332136792 2024-11-11T13:35:41,742 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 495fd2ab746b448abbfb64b606863a6f, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731332137017 2024-11-11T13:35:41,742 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e1ebaf4c4f942498d571537dc6cafc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731332137017 2024-11-11T13:35:41,743 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c567fe0b1da3469f90e9df9fabc78c20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731332137689 2024-11-11T13:35:41,743 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c0ad0f4f68e4891b0964b0c28e4ea08, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731332137689 2024-11-11T13:35:41,744 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b3c772763046e3b420520fc59e264c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332139896 2024-11-11T13:35:41,745 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 272f0a868c9c463a80c4269842243755, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332139896 2024-11-11T13:35:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:41,748 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:41,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:35:41,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e337f550711740eb90b6a2aef6fae44d_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332141073/Put/seqid=0 2024-11-11T13:35:41,790 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#B#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:41,790 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/1c4d0b99e2f1496db6bbf983516ee336 is 50, key is test_row_0/B:col10/1731332139896/Put/seqid=0 2024-11-11T13:35:41,800 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:41,816 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111285bb2616e7049e9b08870d8717557fa_e913da51831f130352321c03b81d22d5 store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:41,819 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111285bb2616e7049e9b08870d8717557fa_e913da51831f130352321c03b81d22d5, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:41,819 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111285bb2616e7049e9b08870d8717557fa_e913da51831f130352321c03b81d22d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:41,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332201808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332201814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332201816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332201824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332201825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T13:35:41,840 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:41,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T13:35:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T13:35:41,843 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:41,844 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:41,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:41,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741982_1158 (size=12204) 2024-11-11T13:35:41,854 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:41,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e337f550711740eb90b6a2aef6fae44d_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e337f550711740eb90b6a2aef6fae44d_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:41,861 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/4b7e70fc190041c785c7b8976acb4713, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:41,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/4b7e70fc190041c785c7b8976acb4713 is 175, key is test_row_0/A:col10/1731332141073/Put/seqid=0 2024-11-11T13:35:41,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741983_1159 (size=12241) 2024-11-11T13:35:41,882 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/1c4d0b99e2f1496db6bbf983516ee336 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1c4d0b99e2f1496db6bbf983516ee336 2024-11-11T13:35:41,895 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e913da51831f130352321c03b81d22d5/B of e913da51831f130352321c03b81d22d5 into 1c4d0b99e2f1496db6bbf983516ee336(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:41,895 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:41,896 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/B, priority=12, startTime=1731332141737; duration=0sec 2024-11-11T13:35:41,896 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:41,896 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:B 2024-11-11T13:35:41,896 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:41,898 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:41,898 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/C is initiating minor compaction (all files) 2024-11-11T13:35:41,898 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/C in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:41,898 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/04980b51d3d94715bbfb174a1c4ddbcd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/787f597937564fadbbc0b148676e4430, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b4ca82302248e5ba6cfaf3eb6a184b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/bfaff851454d428e932486af0b7dcf02] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=47.0 K 2024-11-11T13:35:41,899 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 04980b51d3d94715bbfb174a1c4ddbcd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332136792 2024-11-11T13:35:41,899 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 787f597937564fadbbc0b148676e4430, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731332137017 2024-11-11T13:35:41,900 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b4ca82302248e5ba6cfaf3eb6a184b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731332137689 2024-11-11T13:35:41,900 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting bfaff851454d428e932486af0b7dcf02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332139896 2024-11-11T13:35:41,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741985_1161 (size=31005) 2024-11-11T13:35:41,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741984_1160 (size=4469) 2024-11-11T13:35:41,932 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:41,932 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/3264c56f1eb44ea6b9ad06a475312f41 is 50, key is test_row_0/C:col10/1731332139896/Put/seqid=0 2024-11-11T13:35:41,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332201929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,934 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#A#compaction#137 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332201929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,935 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/29c3be909230453aa921240f6174f237 is 175, key is test_row_0/A:col10/1731332139896/Put/seqid=0 2024-11-11T13:35:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332201929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332201940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332201940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T13:35:41,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741986_1162 (size=12241) 2024-11-11T13:35:41,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741987_1163 (size=31195) 2024-11-11T13:35:41,972 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/3264c56f1eb44ea6b9ad06a475312f41 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/3264c56f1eb44ea6b9ad06a475312f41 2024-11-11T13:35:41,979 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/29c3be909230453aa921240f6174f237 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/29c3be909230453aa921240f6174f237 2024-11-11T13:35:41,988 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e913da51831f130352321c03b81d22d5/C of e913da51831f130352321c03b81d22d5 into 3264c56f1eb44ea6b9ad06a475312f41(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:41,988 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:41,988 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/C, priority=12, startTime=1731332141738; duration=0sec 2024-11-11T13:35:41,988 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:41,988 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:C 2024-11-11T13:35:41,995 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e913da51831f130352321c03b81d22d5/A of e913da51831f130352321c03b81d22d5 into 29c3be909230453aa921240f6174f237(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:41,995 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:41,995 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/A, priority=12, startTime=1731332141737; duration=0sec 2024-11-11T13:35:41,995 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:41,995 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:A 2024-11-11T13:35:42,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T13:35:42,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:42,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:42,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:42,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:42,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:42,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332202141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332202142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332202143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T13:35:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:42,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:42,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332202150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332202150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T13:35:42,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T13:35:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:42,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:42,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:42,334 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/4b7e70fc190041c785c7b8976acb4713 2024-11-11T13:35:42,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/e5ac53a722c64cd38cbc2e2f0b97bc0c is 50, key is test_row_0/B:col10/1731332141073/Put/seqid=0 2024-11-11T13:35:42,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741988_1164 (size=12051) 2024-11-11T13:35:42,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/e5ac53a722c64cd38cbc2e2f0b97bc0c 2024-11-11T13:35:42,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/4b0876a3161a43b8a3a75431e07357d6 is 50, key is test_row_0/C:col10/1731332141073/Put/seqid=0 2024-11-11T13:35:42,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741989_1165 (size=12051) 2024-11-11T13:35:42,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/4b0876a3161a43b8a3a75431e07357d6 2024-11-11T13:35:42,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/4b7e70fc190041c785c7b8976acb4713 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/4b7e70fc190041c785c7b8976acb4713 2024-11-11T13:35:42,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/4b7e70fc190041c785c7b8976acb4713, entries=150, sequenceid=131, filesize=30.3 K 2024-11-11T13:35:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/e5ac53a722c64cd38cbc2e2f0b97bc0c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e5ac53a722c64cd38cbc2e2f0b97bc0c 2024-11-11T13:35:42,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e5ac53a722c64cd38cbc2e2f0b97bc0c, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T13:35:42,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/4b0876a3161a43b8a3a75431e07357d6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/4b0876a3161a43b8a3a75431e07357d6 2024-11-11T13:35:42,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/4b0876a3161a43b8a3a75431e07357d6, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T13:35:42,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e913da51831f130352321c03b81d22d5 in 713ms, sequenceid=131, compaction requested=false 2024-11-11T13:35:42,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:42,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T13:35:42,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:42,468 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-11T13:35:42,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:42,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d7dc2b55bd89427a8bdcd8460bfa823c_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332141821/Put/seqid=0 2024-11-11T13:35:42,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332202473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332202479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332202479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332202479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T13:35:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332202480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741990_1166 (size=14794) 2024-11-11T13:35:42,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332202589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332202589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332202589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332202799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332202799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332202799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:42,934 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d7dc2b55bd89427a8bdcd8460bfa823c_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d7dc2b55bd89427a8bdcd8460bfa823c_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/df176d1b5bf44963a41878a5fd02f25a, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:42,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/df176d1b5bf44963a41878a5fd02f25a is 175, key is test_row_0/A:col10/1731332141821/Put/seqid=0 2024-11-11T13:35:42,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741991_1167 (size=39749) 2024-11-11T13:35:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T13:35:42,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332202990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:42,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332202991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332203109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332203109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332203109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,358 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/df176d1b5bf44963a41878a5fd02f25a 2024-11-11T13:35:43,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/283099fcfca94107b21d4430bc64a589 is 50, key is test_row_0/B:col10/1731332141821/Put/seqid=0 2024-11-11T13:35:43,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741992_1168 (size=12151) 2024-11-11T13:35:43,428 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/283099fcfca94107b21d4430bc64a589 2024-11-11T13:35:43,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/b759ba23e2f74ac88e6f16422755c8ff is 50, key is test_row_0/C:col10/1731332141821/Put/seqid=0 2024-11-11T13:35:43,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741993_1169 (size=12151) 2024-11-11T13:35:43,527 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/b759ba23e2f74ac88e6f16422755c8ff 2024-11-11T13:35:43,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/df176d1b5bf44963a41878a5fd02f25a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/df176d1b5bf44963a41878a5fd02f25a 2024-11-11T13:35:43,552 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/df176d1b5bf44963a41878a5fd02f25a, entries=200, sequenceid=163, filesize=38.8 K 2024-11-11T13:35:43,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/283099fcfca94107b21d4430bc64a589 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/283099fcfca94107b21d4430bc64a589 2024-11-11T13:35:43,563 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/283099fcfca94107b21d4430bc64a589, entries=150, sequenceid=163, filesize=11.9 K 2024-11-11T13:35:43,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/b759ba23e2f74ac88e6f16422755c8ff as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/b759ba23e2f74ac88e6f16422755c8ff 2024-11-11T13:35:43,580 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/b759ba23e2f74ac88e6f16422755c8ff, entries=150, sequenceid=163, filesize=11.9 K 2024-11-11T13:35:43,581 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for e913da51831f130352321c03b81d22d5 in 1113ms, sequenceid=163, compaction requested=true 2024-11-11T13:35:43,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:43,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:43,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T13:35:43,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T13:35:43,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T13:35:43,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7390 sec 2024-11-11T13:35:43,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7460 sec 2024-11-11T13:35:43,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:43,625 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:43,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:35:43,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114cf321a579034b7886a15bcb844489d0_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332143622/Put/seqid=0 2024-11-11T13:35:43,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741994_1170 (size=14794) 2024-11-11T13:35:43,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332203667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332203668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332203669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332203779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332203779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332203779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332203989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332203990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332203991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T13:35:44,000 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:44,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:44,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332204000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:44,001 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_2' in e913da51831f130352321c03b81d22d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:35:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T13:35:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T13:35:44,003 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:44,004 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:44,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:44,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332204009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:44,010 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in e913da51831f130352321c03b81d22d5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:35:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:44,069 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:44,073 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114cf321a579034b7886a15bcb844489d0_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114cf321a579034b7886a15bcb844489d0_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:44,074 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/9044d5f581b84a34b1b04e611b0e0718, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:44,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/9044d5f581b84a34b1b04e611b0e0718 is 175, key is test_row_0/A:col10/1731332143622/Put/seqid=0 2024-11-11T13:35:44,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741995_1171 (size=39749) 2024-11-11T13:35:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T13:35:44,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T13:35:44,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:44,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332204299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:44,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332204299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:44,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332204300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:44,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T13:35:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T13:35:44,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T13:35:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:44,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,481 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/9044d5f581b84a34b1b04e611b0e0718 2024-11-11T13:35:44,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/1ffd5d87f72148ad86e489b077c7e40d is 50, key is test_row_0/B:col10/1731332143622/Put/seqid=0 2024-11-11T13:35:44,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741996_1172 (size=12151) 2024-11-11T13:35:44,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T13:35:44,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:44,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T13:35:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T13:35:44,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:44,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332204809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:44,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332204809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:44,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332204810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:44,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:44,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/1ffd5d87f72148ad86e489b077c7e40d 2024-11-11T13:35:44,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/939db6ad4c604bee8f7ba71ad94853e1 is 50, key is test_row_0/C:col10/1731332143622/Put/seqid=0 2024-11-11T13:35:44,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741997_1173 (size=12151) 2024-11-11T13:35:44,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/939db6ad4c604bee8f7ba71ad94853e1 2024-11-11T13:35:44,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T13:35:44,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:44,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:44,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:44,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/9044d5f581b84a34b1b04e611b0e0718 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9044d5f581b84a34b1b04e611b0e0718 2024-11-11T13:35:44,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9044d5f581b84a34b1b04e611b0e0718, entries=200, sequenceid=175, filesize=38.8 K 2024-11-11T13:35:44,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/1ffd5d87f72148ad86e489b077c7e40d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1ffd5d87f72148ad86e489b077c7e40d 2024-11-11T13:35:44,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1ffd5d87f72148ad86e489b077c7e40d, entries=150, sequenceid=175, filesize=11.9 K 2024-11-11T13:35:44,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/939db6ad4c604bee8f7ba71ad94853e1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/939db6ad4c604bee8f7ba71ad94853e1 2024-11-11T13:35:44,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/939db6ad4c604bee8f7ba71ad94853e1, entries=150, sequenceid=175, filesize=11.9 K 2024-11-11T13:35:44,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e913da51831f130352321c03b81d22d5 in 1373ms, sequenceid=175, compaction requested=true 2024-11-11T13:35:44,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:44,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:44,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:44,998 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:44,999 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:44,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:44,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:44,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:44,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:45,000 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:45,000 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:45,000 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/B is initiating minor compaction (all files) 2024-11-11T13:35:45,000 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/B in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:45,000 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1c4d0b99e2f1496db6bbf983516ee336, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e5ac53a722c64cd38cbc2e2f0b97bc0c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/283099fcfca94107b21d4430bc64a589, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1ffd5d87f72148ad86e489b077c7e40d] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=47.5 K 2024-11-11T13:35:45,001 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/A is initiating minor compaction (all files) 2024-11-11T13:35:45,001 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/A in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:45,001 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/29c3be909230453aa921240f6174f237, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/4b7e70fc190041c785c7b8976acb4713, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/df176d1b5bf44963a41878a5fd02f25a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9044d5f581b84a34b1b04e611b0e0718] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=138.4 K 2024-11-11T13:35:45,001 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:45,001 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/29c3be909230453aa921240f6174f237, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/4b7e70fc190041c785c7b8976acb4713, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/df176d1b5bf44963a41878a5fd02f25a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9044d5f581b84a34b1b04e611b0e0718] 2024-11-11T13:35:45,001 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c4d0b99e2f1496db6bbf983516ee336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332139896 2024-11-11T13:35:45,002 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e5ac53a722c64cd38cbc2e2f0b97bc0c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731332141073 2024-11-11T13:35:45,002 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29c3be909230453aa921240f6174f237, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332139896 2024-11-11T13:35:45,002 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 283099fcfca94107b21d4430bc64a589, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731332141821 2024-11-11T13:35:45,002 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b7e70fc190041c785c7b8976acb4713, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731332141073 2024-11-11T13:35:45,002 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ffd5d87f72148ad86e489b077c7e40d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731332142476 2024-11-11T13:35:45,003 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting df176d1b5bf44963a41878a5fd02f25a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731332141821 2024-11-11T13:35:45,003 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9044d5f581b84a34b1b04e611b0e0718, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731332142476 2024-11-11T13:35:45,037 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:45,042 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:45,043 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/e3dc97c611f34e33843894c430d397c9 is 50, key is test_row_0/B:col10/1731332143622/Put/seqid=0 2024-11-11T13:35:45,044 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110b93eed4445d45f1829f71a926ef3acd_e913da51831f130352321c03b81d22d5 store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:45,047 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110b93eed4445d45f1829f71a926ef3acd_e913da51831f130352321c03b81d22d5, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:45,047 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110b93eed4445d45f1829f71a926ef3acd_e913da51831f130352321c03b81d22d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:45,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741999_1175 (size=12527) 2024-11-11T13:35:45,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741998_1174 (size=4469) 2024-11-11T13:35:45,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T13:35:45,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:45,113 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T13:35:45,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111149385cc65bb3442eb8e9f6332473eb75_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332143667/Put/seqid=0 2024-11-11T13:35:45,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742000_1176 (size=12304) 2024-11-11T13:35:45,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T13:35:45,147 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111149385cc65bb3442eb8e9f6332473eb75_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111149385cc65bb3442eb8e9f6332473eb75_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:45,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/3c6ffa3cf9c447d79052db1bee583318, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:45,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/3c6ffa3cf9c447d79052db1bee583318 is 175, key is test_row_0/A:col10/1731332143667/Put/seqid=0 2024-11-11T13:35:45,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742001_1177 (size=31105) 2024-11-11T13:35:45,153 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/3c6ffa3cf9c447d79052db1bee583318 2024-11-11T13:35:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/87ded0e8a41b4a559c139f60ed7334cb is 50, key is test_row_0/B:col10/1731332143667/Put/seqid=0 2024-11-11T13:35:45,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742002_1178 (size=12151) 2024-11-11T13:35:45,274 INFO [master/37d26ecf6460:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T13:35:45,275 INFO [master/37d26ecf6460:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T13:35:45,485 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#A#compaction#147 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:45,485 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/ec144db0b5ac455b9fde69e4fe819917 is 175, key is test_row_0/A:col10/1731332143622/Put/seqid=0 2024-11-11T13:35:45,487 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/e3dc97c611f34e33843894c430d397c9 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e3dc97c611f34e33843894c430d397c9 2024-11-11T13:35:45,496 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e913da51831f130352321c03b81d22d5/B of e913da51831f130352321c03b81d22d5 into e3dc97c611f34e33843894c430d397c9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:45,496 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:45,496 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/B, priority=12, startTime=1731332144998; duration=0sec 2024-11-11T13:35:45,496 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:45,496 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:B 2024-11-11T13:35:45,496 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:45,500 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:45,500 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/C is initiating minor compaction (all files) 2024-11-11T13:35:45,500 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/C in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:45,500 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/3264c56f1eb44ea6b9ad06a475312f41, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/4b0876a3161a43b8a3a75431e07357d6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/b759ba23e2f74ac88e6f16422755c8ff, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/939db6ad4c604bee8f7ba71ad94853e1] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=47.5 K 2024-11-11T13:35:45,501 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 3264c56f1eb44ea6b9ad06a475312f41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332139896 2024-11-11T13:35:45,501 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b0876a3161a43b8a3a75431e07357d6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731332141073 2024-11-11T13:35:45,501 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting b759ba23e2f74ac88e6f16422755c8ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731332141821 2024-11-11T13:35:45,502 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 939db6ad4c604bee8f7ba71ad94853e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731332142476 2024-11-11T13:35:45,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742003_1179 (size=31481) 2024-11-11T13:35:45,522 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#C#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:45,523 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/2672dd6f14214e10a2c9e8231b1a9b70 is 50, key is test_row_0/C:col10/1731332143622/Put/seqid=0 2024-11-11T13:35:45,527 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/ec144db0b5ac455b9fde69e4fe819917 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/ec144db0b5ac455b9fde69e4fe819917 2024-11-11T13:35:45,535 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e913da51831f130352321c03b81d22d5/A of e913da51831f130352321c03b81d22d5 into ec144db0b5ac455b9fde69e4fe819917(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:45,535 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:45,535 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/A, priority=12, startTime=1731332144998; duration=0sec 2024-11-11T13:35:45,535 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:45,535 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:A 2024-11-11T13:35:45,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742004_1180 (size=12527) 2024-11-11T13:35:45,625 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/87ded0e8a41b4a559c139f60ed7334cb 2024-11-11T13:35:45,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/61a61bc555944cae8cb399969f8e1857 is 50, key is test_row_0/C:col10/1731332143667/Put/seqid=0 2024-11-11T13:35:45,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742005_1181 (size=12151) 2024-11-11T13:35:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:45,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:45,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332205852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:45,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332205854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:45,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332205860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:45,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:45,974 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/2672dd6f14214e10a2c9e8231b1a9b70 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/2672dd6f14214e10a2c9e8231b1a9b70 2024-11-11T13:35:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332205970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:45,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332205975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:45,981 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e913da51831f130352321c03b81d22d5/C of e913da51831f130352321c03b81d22d5 into 2672dd6f14214e10a2c9e8231b1a9b70(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:45,982 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:45,982 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/C, priority=12, startTime=1731332144999; duration=0sec 2024-11-11T13:35:45,982 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:45,982 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:C 2024-11-11T13:35:45,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:45,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332205975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332206010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,011 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_2' in e913da51831f130352321c03b81d22d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:35:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332206019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,027 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in e913da51831f130352321c03b81d22d5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:35:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,116 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/61a61bc555944cae8cb399969f8e1857 2024-11-11T13:35:46,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/3c6ffa3cf9c447d79052db1bee583318 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/3c6ffa3cf9c447d79052db1bee583318 2024-11-11T13:35:46,146 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/3c6ffa3cf9c447d79052db1bee583318, entries=150, sequenceid=199, filesize=30.4 K 2024-11-11T13:35:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T13:35:46,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/87ded0e8a41b4a559c139f60ed7334cb as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/87ded0e8a41b4a559c139f60ed7334cb 2024-11-11T13:35:46,158 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/87ded0e8a41b4a559c139f60ed7334cb, entries=150, sequenceid=199, filesize=11.9 K 2024-11-11T13:35:46,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/61a61bc555944cae8cb399969f8e1857 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/61a61bc555944cae8cb399969f8e1857 2024-11-11T13:35:46,168 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/61a61bc555944cae8cb399969f8e1857, entries=150, sequenceid=199, filesize=11.9 K 2024-11-11T13:35:46,169 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e913da51831f130352321c03b81d22d5 in 1056ms, sequenceid=199, compaction requested=false 2024-11-11T13:35:46,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:46,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:46,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T13:35:46,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T13:35:46,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T13:35:46,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-11-11T13:35:46,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.1760 sec 2024-11-11T13:35:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:46,192 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:46,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T13:35:46,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fe0e2129ca1b4bc593f1a294552fa355_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332145846/Put/seqid=0 2024-11-11T13:35:46,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742006_1182 (size=12304) 2024-11-11T13:35:46,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332206265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332206267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332206268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332206379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332206379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332206390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332206590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332206590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332206599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,641 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:46,649 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fe0e2129ca1b4bc593f1a294552fa355_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fe0e2129ca1b4bc593f1a294552fa355_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:46,650 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a7f9bef18d294a4a8acd38e8fd96d58b, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:46,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a7f9bef18d294a4a8acd38e8fd96d58b is 175, key is test_row_0/A:col10/1731332145846/Put/seqid=0 2024-11-11T13:35:46,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742007_1183 (size=31105) 2024-11-11T13:35:46,680 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a7f9bef18d294a4a8acd38e8fd96d58b 2024-11-11T13:35:46,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/79c815300e1f4841aadb5fb613d03208 is 50, key is test_row_0/B:col10/1731332145846/Put/seqid=0 2024-11-11T13:35:46,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742008_1184 (size=12151) 2024-11-11T13:35:46,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332206900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332206900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:46,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332206909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:47,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/79c815300e1f4841aadb5fb613d03208 2024-11-11T13:35:47,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/56658159702f435bb804390e593d73bf is 50, key is test_row_0/C:col10/1731332145846/Put/seqid=0 2024-11-11T13:35:47,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742009_1185 (size=12151) 2024-11-11T13:35:47,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/56658159702f435bb804390e593d73bf 2024-11-11T13:35:47,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a7f9bef18d294a4a8acd38e8fd96d58b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a7f9bef18d294a4a8acd38e8fd96d58b 2024-11-11T13:35:47,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a7f9bef18d294a4a8acd38e8fd96d58b, entries=150, sequenceid=215, filesize=30.4 K 2024-11-11T13:35:47,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/79c815300e1f4841aadb5fb613d03208 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/79c815300e1f4841aadb5fb613d03208 2024-11-11T13:35:47,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/79c815300e1f4841aadb5fb613d03208, entries=150, sequenceid=215, filesize=11.9 K 2024-11-11T13:35:47,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/56658159702f435bb804390e593d73bf as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/56658159702f435bb804390e593d73bf 2024-11-11T13:35:47,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/56658159702f435bb804390e593d73bf, entries=150, sequenceid=215, filesize=11.9 K 2024-11-11T13:35:47,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e913da51831f130352321c03b81d22d5 in 1014ms, sequenceid=215, compaction requested=true 2024-11-11T13:35:47,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:47,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:47,207 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:47,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:47,207 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:47,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:47,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:47,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:47,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:47,209 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93691 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:47,209 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/A is initiating minor compaction (all files) 2024-11-11T13:35:47,209 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:47,209 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/A in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:47,209 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/B is initiating minor compaction (all files) 2024-11-11T13:35:47,209 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/B in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:47,209 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/ec144db0b5ac455b9fde69e4fe819917, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/3c6ffa3cf9c447d79052db1bee583318, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a7f9bef18d294a4a8acd38e8fd96d58b] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=91.5 K 2024-11-11T13:35:47,210 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e3dc97c611f34e33843894c430d397c9, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/87ded0e8a41b4a559c139f60ed7334cb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/79c815300e1f4841aadb5fb613d03208] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=36.0 K 2024-11-11T13:35:47,209 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:47,210 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/ec144db0b5ac455b9fde69e4fe819917, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/3c6ffa3cf9c447d79052db1bee583318, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a7f9bef18d294a4a8acd38e8fd96d58b] 2024-11-11T13:35:47,210 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e3dc97c611f34e33843894c430d397c9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731332142476 2024-11-11T13:35:47,210 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec144db0b5ac455b9fde69e4fe819917, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731332142476 2024-11-11T13:35:47,212 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 87ded0e8a41b4a559c139f60ed7334cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731332143650 2024-11-11T13:35:47,212 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c6ffa3cf9c447d79052db1bee583318, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731332143650 2024-11-11T13:35:47,213 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 79c815300e1f4841aadb5fb613d03208, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731332145846 2024-11-11T13:35:47,213 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7f9bef18d294a4a8acd38e8fd96d58b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731332145846 2024-11-11T13:35:47,246 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:47,246 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/4c71c5c2eba04b2cbfe4ee047a455412 is 50, key is test_row_0/B:col10/1731332145846/Put/seqid=0 2024-11-11T13:35:47,249 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:47,261 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111ad5b59f173444f4e9d7b6221c7015afb_e913da51831f130352321c03b81d22d5 store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:47,263 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111ad5b59f173444f4e9d7b6221c7015afb_e913da51831f130352321c03b81d22d5, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:47,263 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ad5b59f173444f4e9d7b6221c7015afb_e913da51831f130352321c03b81d22d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:47,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742010_1186 (size=12629) 2024-11-11T13:35:47,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742011_1187 (size=4469) 2024-11-11T13:35:47,291 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#A#compaction#157 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:47,292 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/b014dc67ab57443b8e9a63dcbbeda0a7 is 175, key is test_row_0/A:col10/1731332145846/Put/seqid=0 2024-11-11T13:35:47,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742012_1188 (size=31583) 2024-11-11T13:35:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:47,414 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:47,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T13:35:47,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116c3409dcd3424118a3ea7d0f3490d2b3_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332146266/Put/seqid=0 2024-11-11T13:35:47,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332207431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:47,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332207432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:47,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332207437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:47,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742013_1189 (size=14794) 2024-11-11T13:35:47,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332207539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:47,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332207549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:47,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332207549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:47,690 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/4c71c5c2eba04b2cbfe4ee047a455412 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/4c71c5c2eba04b2cbfe4ee047a455412 2024-11-11T13:35:47,695 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e913da51831f130352321c03b81d22d5/B of e913da51831f130352321c03b81d22d5 into 4c71c5c2eba04b2cbfe4ee047a455412(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:47,696 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:47,696 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/B, priority=13, startTime=1731332147207; duration=0sec 2024-11-11T13:35:47,696 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:47,696 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:B 2024-11-11T13:35:47,696 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:47,697 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:47,697 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/C is initiating minor compaction (all files) 2024-11-11T13:35:47,697 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/C in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:47,697 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/2672dd6f14214e10a2c9e8231b1a9b70, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/61a61bc555944cae8cb399969f8e1857, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/56658159702f435bb804390e593d73bf] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=36.0 K 2024-11-11T13:35:47,698 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2672dd6f14214e10a2c9e8231b1a9b70, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731332142476 2024-11-11T13:35:47,698 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 61a61bc555944cae8cb399969f8e1857, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731332143650 2024-11-11T13:35:47,699 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 56658159702f435bb804390e593d73bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731332145846 2024-11-11T13:35:47,703 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/b014dc67ab57443b8e9a63dcbbeda0a7 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/b014dc67ab57443b8e9a63dcbbeda0a7 2024-11-11T13:35:47,710 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e913da51831f130352321c03b81d22d5/A of e913da51831f130352321c03b81d22d5 into b014dc67ab57443b8e9a63dcbbeda0a7(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:47,710 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:47,711 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/A, priority=13, startTime=1731332147207; duration=0sec 2024-11-11T13:35:47,711 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:47,711 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:A 2024-11-11T13:35:47,712 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:47,712 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/8cc38a2528c3437290ec714ca73fdbba is 50, key is test_row_0/C:col10/1731332145846/Put/seqid=0 2024-11-11T13:35:47,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742014_1190 (size=12629) 2024-11-11T13:35:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332207749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:47,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332207759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:47,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332207759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:47,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:47,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:47,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:47,841 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:47,845 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116c3409dcd3424118a3ea7d0f3490d2b3_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116c3409dcd3424118a3ea7d0f3490d2b3_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:47,846 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a39dd93c92ae4e1a94d288ba114e40ca, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:47,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a39dd93c92ae4e1a94d288ba114e40ca is 175, key is test_row_0/A:col10/1731332146266/Put/seqid=0 2024-11-11T13:35:47,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742015_1191 (size=39749) 2024-11-11T13:35:48,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332208059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:48,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332208069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332208069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:48,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:48,122 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/8cc38a2528c3437290ec714ca73fdbba as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/8cc38a2528c3437290ec714ca73fdbba 2024-11-11T13:35:48,128 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e913da51831f130352321c03b81d22d5/C of e913da51831f130352321c03b81d22d5 into 8cc38a2528c3437290ec714ca73fdbba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:48,128 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:48,128 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/C, priority=13, startTime=1731332147207; duration=0sec 2024-11-11T13:35:48,128 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:48,128 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:C 2024-11-11T13:35:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T13:35:48,160 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:48,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T13:35:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T13:35:48,162 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:48,162 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:48,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:48,252 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a39dd93c92ae4e1a94d288ba114e40ca 2024-11-11T13:35:48,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/0b25f69cfb4b47dd8da10c05fd923a37 is 50, key is test_row_0/B:col10/1731332146266/Put/seqid=0 2024-11-11T13:35:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T13:35:48,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742016_1192 (size=12151) 2024-11-11T13:35:48,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T13:35:48,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:48,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:48,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:48,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T13:35:48,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:48,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:48,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:48,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T13:35:48,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332208570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:48,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332208579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332208579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:48,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:48,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T13:35:48,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:48,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:48,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:48,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/0b25f69cfb4b47dd8da10c05fd923a37 2024-11-11T13:35:48,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/97fbc54fb5784709ad9475642aa9a613 is 50, key is test_row_0/C:col10/1731332146266/Put/seqid=0 2024-11-11T13:35:48,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742017_1193 (size=12151) 2024-11-11T13:35:48,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T13:35:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:48,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T13:35:48,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T13:35:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:48,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:49,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T13:35:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:49,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:49,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:49,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/97fbc54fb5784709ad9475642aa9a613 2024-11-11T13:35:49,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a39dd93c92ae4e1a94d288ba114e40ca as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a39dd93c92ae4e1a94d288ba114e40ca 2024-11-11T13:35:49,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a39dd93c92ae4e1a94d288ba114e40ca, entries=200, sequenceid=240, filesize=38.8 K 2024-11-11T13:35:49,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/0b25f69cfb4b47dd8da10c05fd923a37 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/0b25f69cfb4b47dd8da10c05fd923a37 2024-11-11T13:35:49,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/0b25f69cfb4b47dd8da10c05fd923a37, entries=150, sequenceid=240, filesize=11.9 K 2024-11-11T13:35:49,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/97fbc54fb5784709ad9475642aa9a613 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/97fbc54fb5784709ad9475642aa9a613 2024-11-11T13:35:49,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/97fbc54fb5784709ad9475642aa9a613, entries=150, sequenceid=240, filesize=11.9 K 2024-11-11T13:35:49,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e913da51831f130352321c03b81d22d5 in 1720ms, sequenceid=240, compaction requested=false 2024-11-11T13:35:49,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:49,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T13:35:49,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:49,235 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:35:49,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a4f169f5fa6b4464a0435c0c1f8b6159_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332147435/Put/seqid=0 2024-11-11T13:35:49,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742018_1194 (size=12304) 2024-11-11T13:35:49,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:49,255 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a4f169f5fa6b4464a0435c0c1f8b6159_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a4f169f5fa6b4464a0435c0c1f8b6159_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:49,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/da83647d7a8644ccbb3f4627b576ea35, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:49,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/da83647d7a8644ccbb3f4627b576ea35 is 175, key is test_row_0/A:col10/1731332147435/Put/seqid=0 2024-11-11T13:35:49,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742019_1195 (size=31105) 2024-11-11T13:35:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T13:35:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:49,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:49,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332209620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:49,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332209621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:49,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332209621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:49,672 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/da83647d7a8644ccbb3f4627b576ea35 2024-11-11T13:35:49,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/d9127ee163de4a7fae6560a053794f56 is 50, key is test_row_0/B:col10/1731332147435/Put/seqid=0 2024-11-11T13:35:49,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742020_1196 (size=12151) 2024-11-11T13:35:49,707 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/d9127ee163de4a7fae6560a053794f56 2024-11-11T13:35:49,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/d3b0d374493b49e89017f282d07a6989 is 50, key is test_row_0/C:col10/1731332147435/Put/seqid=0 2024-11-11T13:35:49,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332209729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:49,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742021_1197 (size=12151) 2024-11-11T13:35:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332209735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:49,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332209740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:49,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332209939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:49,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332209949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:49,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332209949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33986 deadline: 1731332210039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,040 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_2' in e913da51831f130352321c03b81d22d5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:35:50,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33974 deadline: 1731332210070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,073 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in e913da51831f130352321c03b81d22d5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:35:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,135 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/d3b0d374493b49e89017f282d07a6989 2024-11-11T13:35:50,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/da83647d7a8644ccbb3f4627b576ea35 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/da83647d7a8644ccbb3f4627b576ea35 2024-11-11T13:35:50,145 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/da83647d7a8644ccbb3f4627b576ea35, entries=150, sequenceid=254, filesize=30.4 K 2024-11-11T13:35:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/d9127ee163de4a7fae6560a053794f56 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d9127ee163de4a7fae6560a053794f56 2024-11-11T13:35:50,151 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d9127ee163de4a7fae6560a053794f56, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T13:35:50,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/d3b0d374493b49e89017f282d07a6989 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b0d374493b49e89017f282d07a6989 2024-11-11T13:35:50,158 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b0d374493b49e89017f282d07a6989, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T13:35:50,159 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e913da51831f130352321c03b81d22d5 in 924ms, sequenceid=254, compaction requested=true 2024-11-11T13:35:50,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:50,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:50,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T13:35:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T13:35:50,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T13:35:50,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9980 sec 2024-11-11T13:35:50,164 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.0030 sec 2024-11-11T13:35:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:50,255 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:50,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T13:35:50,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111104eac30d20e34b639e7fdc85798f3373_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332149619/Put/seqid=0 2024-11-11T13:35:50,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742022_1198 (size=14994) 2024-11-11T13:35:50,278 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:50,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332210272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332210273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332210278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111104eac30d20e34b639e7fdc85798f3373_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111104eac30d20e34b639e7fdc85798f3373_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:50,285 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/02af8222a8f346bf9c51dc2d3656f9a1, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:50,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/02af8222a8f346bf9c51dc2d3656f9a1 is 175, key is test_row_0/A:col10/1731332149619/Put/seqid=0 2024-11-11T13:35:50,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742023_1199 (size=39949) 2024-11-11T13:35:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T13:35:50,309 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:50,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:50,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T13:35:50,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T13:35:50,313 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:50,314 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:50,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:50,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332210390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332210390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332210390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T13:35:50,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T13:35:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:50,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332210600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332210600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332210600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T13:35:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:50,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T13:35:50,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/02af8222a8f346bf9c51dc2d3656f9a1 2024-11-11T13:35:50,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/b338d75cfe13483a80ba3d229adfa80c is 50, key is test_row_0/B:col10/1731332149619/Put/seqid=0 2024-11-11T13:35:50,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742024_1200 (size=12301) 2024-11-11T13:35:50,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T13:35:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:50,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332210909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332210909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332210910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:50,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:50,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T13:35:50,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:50,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:50,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:50,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T13:35:51,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T13:35:51,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:51,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:51,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:51,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:51,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:51,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/b338d75cfe13483a80ba3d229adfa80c 2024-11-11T13:35:51,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/a8b18df2ab144a289c6603a9c6c3c19d is 50, key is test_row_0/C:col10/1731332149619/Put/seqid=0 2024-11-11T13:35:51,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742025_1201 (size=12301) 2024-11-11T13:35:51,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/a8b18df2ab144a289c6603a9c6c3c19d 2024-11-11T13:35:51,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/02af8222a8f346bf9c51dc2d3656f9a1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/02af8222a8f346bf9c51dc2d3656f9a1 2024-11-11T13:35:51,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/02af8222a8f346bf9c51dc2d3656f9a1, entries=200, sequenceid=280, filesize=39.0 K 2024-11-11T13:35:51,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/b338d75cfe13483a80ba3d229adfa80c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b338d75cfe13483a80ba3d229adfa80c 2024-11-11T13:35:51,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b338d75cfe13483a80ba3d229adfa80c, entries=150, sequenceid=280, filesize=12.0 K 2024-11-11T13:35:51,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/a8b18df2ab144a289c6603a9c6c3c19d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/a8b18df2ab144a289c6603a9c6c3c19d 2024-11-11T13:35:51,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T13:35:51,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:51,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:51,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:51,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:51,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:51,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/a8b18df2ab144a289c6603a9c6c3c19d, entries=150, sequenceid=280, filesize=12.0 K 2024-11-11T13:35:51,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e913da51831f130352321c03b81d22d5 in 995ms, sequenceid=280, compaction requested=true 2024-11-11T13:35:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:51,250 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:35:51,250 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:51,252 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:51,252 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/B is initiating minor compaction (all files) 2024-11-11T13:35:51,252 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/B in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:51,253 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/4c71c5c2eba04b2cbfe4ee047a455412, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/0b25f69cfb4b47dd8da10c05fd923a37, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d9127ee163de4a7fae6560a053794f56, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b338d75cfe13483a80ba3d229adfa80c] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=48.1 K 2024-11-11T13:35:51,253 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142386 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:51,253 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/A is initiating minor compaction (all files) 2024-11-11T13:35:51,253 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/A in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:51,253 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/b014dc67ab57443b8e9a63dcbbeda0a7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a39dd93c92ae4e1a94d288ba114e40ca, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/da83647d7a8644ccbb3f4627b576ea35, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/02af8222a8f346bf9c51dc2d3656f9a1] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=139.0 K 2024-11-11T13:35:51,253 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:51,254 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/b014dc67ab57443b8e9a63dcbbeda0a7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a39dd93c92ae4e1a94d288ba114e40ca, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/da83647d7a8644ccbb3f4627b576ea35, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/02af8222a8f346bf9c51dc2d3656f9a1] 2024-11-11T13:35:51,254 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c71c5c2eba04b2cbfe4ee047a455412, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731332145846 2024-11-11T13:35:51,255 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b25f69cfb4b47dd8da10c05fd923a37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731332146266 2024-11-11T13:35:51,255 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting b014dc67ab57443b8e9a63dcbbeda0a7, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731332145846 2024-11-11T13:35:51,256 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting a39dd93c92ae4e1a94d288ba114e40ca, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731332146264 2024-11-11T13:35:51,256 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9127ee163de4a7fae6560a053794f56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731332147422 2024-11-11T13:35:51,256 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting b338d75cfe13483a80ba3d229adfa80c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731332149618 2024-11-11T13:35:51,256 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting da83647d7a8644ccbb3f4627b576ea35, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731332147422 2024-11-11T13:35:51,258 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 02af8222a8f346bf9c51dc2d3656f9a1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731332149618 2024-11-11T13:35:51,292 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:51,294 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#B#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:51,295 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/eac21cc5c38b4fbab8c6c02468a079b1 is 50, key is test_row_0/B:col10/1731332149619/Put/seqid=0 2024-11-11T13:35:51,300 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111eea17d0aa9184c129a9a8c59eef57207_e913da51831f130352321c03b81d22d5 store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:51,303 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111eea17d0aa9184c129a9a8c59eef57207_e913da51831f130352321c03b81d22d5, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:51,303 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111eea17d0aa9184c129a9a8c59eef57207_e913da51831f130352321c03b81d22d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:51,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742026_1202 (size=12915) 2024-11-11T13:35:51,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742027_1203 (size=4469) 2024-11-11T13:35:51,325 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#A#compaction#169 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:51,326 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/d5f6b335bf13471e957ec7e6f4c15651 is 175, key is test_row_0/A:col10/1731332149619/Put/seqid=0 2024-11-11T13:35:51,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742028_1204 (size=31869) 2024-11-11T13:35:51,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T13:35:51,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:51,396 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-11T13:35:51,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116eb37c41911b4cd19bfd100efadef1e1_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332150276/Put/seqid=0 2024-11-11T13:35:51,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742029_1205 (size=12454) 2024-11-11T13:35:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:51,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T13:35:51,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332211464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:51,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332211465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:51,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332211467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:51,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332211573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:51,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332211581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:51,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332211581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:51,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:51,754 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/eac21cc5c38b4fbab8c6c02468a079b1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/eac21cc5c38b4fbab8c6c02468a079b1 2024-11-11T13:35:51,760 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e913da51831f130352321c03b81d22d5/B of e913da51831f130352321c03b81d22d5 into eac21cc5c38b4fbab8c6c02468a079b1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:51,760 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:51,760 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/B, priority=12, startTime=1731332151250; duration=0sec 2024-11-11T13:35:51,761 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:51,761 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:B 2024-11-11T13:35:51,761 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/d5f6b335bf13471e957ec7e6f4c15651 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/d5f6b335bf13471e957ec7e6f4c15651 2024-11-11T13:35:51,761 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:35:51,763 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:35:51,763 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/C is initiating minor compaction (all files) 2024-11-11T13:35:51,763 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/C in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:51,763 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/8cc38a2528c3437290ec714ca73fdbba, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/97fbc54fb5784709ad9475642aa9a613, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b0d374493b49e89017f282d07a6989, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/a8b18df2ab144a289c6603a9c6c3c19d] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=48.1 K 2024-11-11T13:35:51,764 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cc38a2528c3437290ec714ca73fdbba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731332145846 2024-11-11T13:35:51,766 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97fbc54fb5784709ad9475642aa9a613, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731332146266 2024-11-11T13:35:51,767 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b0d374493b49e89017f282d07a6989, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731332147422 2024-11-11T13:35:51,767 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8b18df2ab144a289c6603a9c6c3c19d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731332149618 2024-11-11T13:35:51,772 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e913da51831f130352321c03b81d22d5/A of e913da51831f130352321c03b81d22d5 into d5f6b335bf13471e957ec7e6f4c15651(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:51,772 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:51,772 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/A, priority=12, startTime=1731332151250; duration=0sec 2024-11-11T13:35:51,773 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:51,773 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:A 2024-11-11T13:35:51,780 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#C#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:51,781 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/30a0099b875a448988625c776fbcabcc is 50, key is test_row_0/C:col10/1731332149619/Put/seqid=0 2024-11-11T13:35:51,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332211789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:51,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332211789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:51,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332211789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:51,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:51,815 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116eb37c41911b4cd19bfd100efadef1e1_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116eb37c41911b4cd19bfd100efadef1e1_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:51,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/9877b3e336634f17aaa417898729b3bd, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:51,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/9877b3e336634f17aaa417898729b3bd is 175, key is test_row_0/A:col10/1731332150276/Put/seqid=0 2024-11-11T13:35:51,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742030_1206 (size=12915) 2024-11-11T13:35:51,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742031_1207 (size=31255) 2024-11-11T13:35:52,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332212100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332212100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:52,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332212100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:52,225 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/30a0099b875a448988625c776fbcabcc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/30a0099b875a448988625c776fbcabcc 2024-11-11T13:35:52,230 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e913da51831f130352321c03b81d22d5/C of e913da51831f130352321c03b81d22d5 into 30a0099b875a448988625c776fbcabcc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:52,230 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:52,230 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/C, priority=12, startTime=1731332151250; duration=0sec 2024-11-11T13:35:52,230 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:52,230 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:C 2024-11-11T13:35:52,237 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/9877b3e336634f17aaa417898729b3bd 2024-11-11T13:35:52,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/d7896eee9e9740b889ddccc48aac3cda is 50, key is test_row_0/B:col10/1731332150276/Put/seqid=0 2024-11-11T13:35:52,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742032_1208 (size=12301) 2024-11-11T13:35:52,301 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/d7896eee9e9740b889ddccc48aac3cda 2024-11-11T13:35:52,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/3e7dd916060b4bab80f61ea72dde01a8 is 50, key is test_row_0/C:col10/1731332150276/Put/seqid=0 2024-11-11T13:35:52,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742033_1209 (size=12301) 2024-11-11T13:35:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T13:35:52,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332212610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:52,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332212610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:52,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332212610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:52,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:52,736 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/3e7dd916060b4bab80f61ea72dde01a8 2024-11-11T13:35:52,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/9877b3e336634f17aaa417898729b3bd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9877b3e336634f17aaa417898729b3bd 2024-11-11T13:35:52,832 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9877b3e336634f17aaa417898729b3bd, entries=150, sequenceid=290, filesize=30.5 K 2024-11-11T13:35:52,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/d7896eee9e9740b889ddccc48aac3cda as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d7896eee9e9740b889ddccc48aac3cda 2024-11-11T13:35:52,871 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d7896eee9e9740b889ddccc48aac3cda, entries=150, sequenceid=290, filesize=12.0 K 2024-11-11T13:35:52,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/3e7dd916060b4bab80f61ea72dde01a8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/3e7dd916060b4bab80f61ea72dde01a8 2024-11-11T13:35:52,893 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/3e7dd916060b4bab80f61ea72dde01a8, entries=150, sequenceid=290, filesize=12.0 K 2024-11-11T13:35:52,895 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e913da51831f130352321c03b81d22d5 in 1498ms, sequenceid=290, compaction requested=false 2024-11-11T13:35:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T13:35:52,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T13:35:52,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T13:35:52,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5820 sec 2024-11-11T13:35:52,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.5880 sec 2024-11-11T13:35:53,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on e913da51831f130352321c03b81d22d5 2024-11-11T13:35:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:35:53,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-11T13:35:53,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e18f8eb66277424e91d4a54434df3a37_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332153621/Put/seqid=0 2024-11-11T13:35:53,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742034_1210 (size=14994) 2024-11-11T13:35:53,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:53,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332213683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:53,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332213683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:53,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332213691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:53,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:53,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332213800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332213800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332213800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:53,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:54,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332214011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332214011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:54,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332214011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:54,052 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:54,061 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e18f8eb66277424e91d4a54434df3a37_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e18f8eb66277424e91d4a54434df3a37_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:54,066 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/c24ef28bac81461280ad32628e9a7444, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:54,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/c24ef28bac81461280ad32628e9a7444 is 175, key is test_row_0/A:col10/1731332153621/Put/seqid=0 2024-11-11T13:35:54,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742035_1211 (size=39949) 2024-11-11T13:35:54,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332214319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:54,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332214320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:54,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:54,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332214323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T13:35:54,481 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:54,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:35:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-11T13:35:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T13:35:54,489 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:35:54,490 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:35:54,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:35:54,497 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/c24ef28bac81461280ad32628e9a7444 2024-11-11T13:35:54,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/54bf7425de9d42769b462efe3926c2b4 is 50, key is test_row_0/B:col10/1731332153621/Put/seqid=0 2024-11-11T13:35:54,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742036_1212 (size=12301) 2024-11-11T13:35:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T13:35:54,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T13:35:54,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:54,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:54,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:54,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:54,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:54,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T13:35:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:54,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T13:35:54,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33950 deadline: 1731332214830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:54,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33952 deadline: 1731332214831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:54,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:35:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33944 deadline: 1731332214831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:35:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:35:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:35:54,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/54bf7425de9d42769b462efe3926c2b4 2024-11-11T13:35:54,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/d7e97e34adcc4cd6a47cc39a64fc64fd is 50, key is test_row_0/C:col10/1731332153621/Put/seqid=0 2024-11-11T13:35:54,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T13:35:54,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:54,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. as already flushing 2024-11-11T13:35:54,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:54,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:35:54,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742037_1213 (size=12301) 2024-11-11T13:35:54,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/d7e97e34adcc4cd6a47cc39a64fc64fd 2024-11-11T13:35:54,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/c24ef28bac81461280ad32628e9a7444 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c24ef28bac81461280ad32628e9a7444 2024-11-11T13:35:55,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c24ef28bac81461280ad32628e9a7444, entries=200, sequenceid=320, filesize=39.0 K 2024-11-11T13:35:55,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/54bf7425de9d42769b462efe3926c2b4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/54bf7425de9d42769b462efe3926c2b4 2024-11-11T13:35:55,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/54bf7425de9d42769b462efe3926c2b4, entries=150, sequenceid=320, filesize=12.0 K 2024-11-11T13:35:55,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/d7e97e34adcc4cd6a47cc39a64fc64fd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d7e97e34adcc4cd6a47cc39a64fc64fd 2024-11-11T13:35:55,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d7e97e34adcc4cd6a47cc39a64fc64fd, entries=150, sequenceid=320, filesize=12.0 K 2024-11-11T13:35:55,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e913da51831f130352321c03b81d22d5 in 1432ms, sequenceid=320, compaction requested=true 2024-11-11T13:35:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:35:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:55,054 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:35:55,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:55,055 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:55,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e913da51831f130352321c03b81d22d5:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:35:55,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:55,056 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:55,056 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:55,056 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/B is initiating minor compaction (all files) 2024-11-11T13:35:55,056 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/A is initiating minor compaction (all files) 2024-11-11T13:35:55,056 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/B in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:55,056 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/A in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:55,056 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/d5f6b335bf13471e957ec7e6f4c15651, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9877b3e336634f17aaa417898729b3bd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c24ef28bac81461280ad32628e9a7444] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=100.7 K 2024-11-11T13:35:55,056 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/eac21cc5c38b4fbab8c6c02468a079b1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d7896eee9e9740b889ddccc48aac3cda, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/54bf7425de9d42769b462efe3926c2b4] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=36.6 K 2024-11-11T13:35:55,056 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:55,056 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/d5f6b335bf13471e957ec7e6f4c15651, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9877b3e336634f17aaa417898729b3bd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c24ef28bac81461280ad32628e9a7444] 2024-11-11T13:35:55,057 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting eac21cc5c38b4fbab8c6c02468a079b1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731332149618 2024-11-11T13:35:55,057 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting d7896eee9e9740b889ddccc48aac3cda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731332150264 2024-11-11T13:35:55,057 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5f6b335bf13471e957ec7e6f4c15651, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731332149618 2024-11-11T13:35:55,060 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 54bf7425de9d42769b462efe3926c2b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731332151462 2024-11-11T13:35:55,061 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9877b3e336634f17aaa417898729b3bd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731332150264 2024-11-11T13:35:55,063 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting c24ef28bac81461280ad32628e9a7444, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731332151462 2024-11-11T13:35:55,070 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:55,075 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111876c031f829046ef96b0c0487bc8ce5e_e913da51831f130352321c03b81d22d5 store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:55,077 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111876c031f829046ef96b0c0487bc8ce5e_e913da51831f130352321c03b81d22d5, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:55,078 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111876c031f829046ef96b0c0487bc8ce5e_e913da51831f130352321c03b81d22d5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:55,084 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:55,085 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/6c7680d89ccf450ab6c45863f17168c7 is 50, key is test_row_0/B:col10/1731332153621/Put/seqid=0 2024-11-11T13:35:55,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742038_1214 (size=4469) 2024-11-11T13:35:55,098 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#A#compaction#177 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:55,099 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a8311f6b3272496daf8cf0e1291e3b55 is 175, key is test_row_0/A:col10/1731332153621/Put/seqid=0 2024-11-11T13:35:55,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742039_1215 (size=13017) 2024-11-11T13:35:55,106 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/6c7680d89ccf450ab6c45863f17168c7 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/6c7680d89ccf450ab6c45863f17168c7 2024-11-11T13:35:55,115 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e913da51831f130352321c03b81d22d5/B of e913da51831f130352321c03b81d22d5 into 6c7680d89ccf450ab6c45863f17168c7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:55,115 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:55,115 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/B, priority=13, startTime=1731332155054; duration=0sec 2024-11-11T13:35:55,115 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:35:55,115 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:B 2024-11-11T13:35:55,115 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:35:55,119 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:35:55,119 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): e913da51831f130352321c03b81d22d5/C is initiating minor compaction (all files) 2024-11-11T13:35:55,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T13:35:55,120 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e913da51831f130352321c03b81d22d5/C in TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:55,120 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/30a0099b875a448988625c776fbcabcc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/3e7dd916060b4bab80f61ea72dde01a8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d7e97e34adcc4cd6a47cc39a64fc64fd] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp, totalSize=36.6 K 2024-11-11T13:35:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T13:35:55,121 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 30a0099b875a448988625c776fbcabcc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731332149618 2024-11-11T13:35:55,122 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e7dd916060b4bab80f61ea72dde01a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731332150264 2024-11-11T13:35:55,122 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e97e34adcc4cd6a47cc39a64fc64fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731332151462 2024-11-11T13:35:55,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:55,123 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-11T13:35:55,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742040_1216 (size=31971) 2024-11-11T13:35:55,146 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/a8311f6b3272496daf8cf0e1291e3b55 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a8311f6b3272496daf8cf0e1291e3b55 2024-11-11T13:35:55,160 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e913da51831f130352321c03b81d22d5/A of e913da51831f130352321c03b81d22d5 into a8311f6b3272496daf8cf0e1291e3b55(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:55,160 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:55,160 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/A, priority=13, startTime=1731332155054; duration=0sec 2024-11-11T13:35:55,160 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:55,160 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:A 2024-11-11T13:35:55,161 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e913da51831f130352321c03b81d22d5#C#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:35:55,161 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/0d15461805944115a3c2324183d4d002 is 50, key is test_row_0/C:col10/1731332153621/Put/seqid=0 2024-11-11T13:35:55,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111e04e1ccf56c4f3ba07fb1ff195b27ff_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332153639/Put/seqid=0 2024-11-11T13:35:55,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742041_1217 (size=13017) 2024-11-11T13:35:55,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742042_1218 (size=12454) 2024-11-11T13:35:55,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,200 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/0d15461805944115a3c2324183d4d002 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/0d15461805944115a3c2324183d4d002 2024-11-11T13:35:55,206 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111e04e1ccf56c4f3ba07fb1ff195b27ff_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111e04e1ccf56c4f3ba07fb1ff195b27ff_e913da51831f130352321c03b81d22d5 2024-11-11T13:35:55,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/113d4567e5b54f50963c3a87a0687130, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:35:55,207 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e913da51831f130352321c03b81d22d5/C of e913da51831f130352321c03b81d22d5 into 0d15461805944115a3c2324183d4d002(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:35:55,207 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:55,207 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., storeName=e913da51831f130352321c03b81d22d5/C, priority=13, startTime=1731332155055; duration=0sec 2024-11-11T13:35:55,207 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:35:55,207 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e913da51831f130352321c03b81d22d5:C 2024-11-11T13:35:55,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/113d4567e5b54f50963c3a87a0687130 is 175, key is test_row_0/A:col10/1731332153639/Put/seqid=0 2024-11-11T13:35:55,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742043_1219 (size=31255) 2024-11-11T13:35:55,223 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/113d4567e5b54f50963c3a87a0687130 2024-11-11T13:35:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/c2c73c49a0a54c8b8314ed3d7243577f is 50, key is test_row_0/B:col10/1731332153639/Put/seqid=0 2024-11-11T13:35:55,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742044_1220 (size=12301) 2024-11-11T13:35:55,264 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/c2c73c49a0a54c8b8314ed3d7243577f 2024-11-11T13:35:55,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/03b05f35fc4d4edea91fdaf2e7d18331 is 50, key is test_row_0/C:col10/1731332153639/Put/seqid=0 2024-11-11T13:35:55,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742045_1221 (size=12301) 2024-11-11T13:35:55,297 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/03b05f35fc4d4edea91fdaf2e7d18331 2024-11-11T13:35:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/113d4567e5b54f50963c3a87a0687130 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/113d4567e5b54f50963c3a87a0687130 2024-11-11T13:35:55,317 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/113d4567e5b54f50963c3a87a0687130, entries=150, sequenceid=330, filesize=30.5 K 2024-11-11T13:35:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/c2c73c49a0a54c8b8314ed3d7243577f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/c2c73c49a0a54c8b8314ed3d7243577f 2024-11-11T13:35:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,326 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/c2c73c49a0a54c8b8314ed3d7243577f, entries=150, sequenceid=330, filesize=12.0 K 2024-11-11T13:35:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/03b05f35fc4d4edea91fdaf2e7d18331 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/03b05f35fc4d4edea91fdaf2e7d18331 2024-11-11T13:35:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,339 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/03b05f35fc4d4edea91fdaf2e7d18331, entries=150, sequenceid=330, filesize=12.0 K 2024-11-11T13:35:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,340 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for e913da51831f130352321c03b81d22d5 in 217ms, sequenceid=330, compaction requested=false 2024-11-11T13:35:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e913da51831f130352321c03b81d22d5: 2024-11-11T13:35:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:35:55,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-11T13:35:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-11T13:35:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T13:35:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 852 msec 2024-11-11T13:35:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 860 msec 2024-11-11T13:35:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,531 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-11T13:35:55,531 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:55,531 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,531 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,531 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,536 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-11T13:35:55,536 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:55,536 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,536 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,566 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-11-11T13:35:55,566 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:55,566 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,566 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:35:55,630 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-11T13:35:55,630 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:55,630 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T13:35:55,630 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,630 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:55,630 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:35:55,841 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-11T13:35:55,841 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:55,841 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,841 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:55,851 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-11T13:35:55,851 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-11T13:35:55,851 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:55,851 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:35:55,851 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,851 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,852 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,852 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:35:55,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:55,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:35:59,756 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T13:36:00,101 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-11T13:36:00,101 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:00,101 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:00,101 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:00,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:00,121 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-11T13:36:00,121 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:00,121 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:00,121 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:00,121 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:00,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T13:36:00,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-11T13:36:00,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-11T13:36:00,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-11T13:36:00,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-11T13:36:00,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-11T13:36:00,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T13:36:00,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2998 2024-11-11T13:36:00,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3000 2024-11-11T13:36:00,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T13:36:00,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1354 2024-11-11T13:36:00,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4062 rows 2024-11-11T13:36:00,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1336 2024-11-11T13:36:00,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4008 rows 2024-11-11T13:36:00,122 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:36:00,122 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:36:00,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:00,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:00,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:00,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T13:36:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T13:36:00,128 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332160127"}]},"ts":"1731332160127"} 2024-11-11T13:36:00,130 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T13:36:00,130 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T13:36:00,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T13:36:00,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, UNASSIGN}] 2024-11-11T13:36:00,133 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, UNASSIGN 2024-11-11T13:36:00,134 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=e913da51831f130352321c03b81d22d5, regionState=CLOSING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:00,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, UNASSIGN because future has completed 2024-11-11T13:36:00,135 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T13:36:00,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:36:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T13:36:00,287 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close e913da51831f130352321c03b81d22d5 2024-11-11T13:36:00,287 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T13:36:00,288 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing e913da51831f130352321c03b81d22d5, disabling compactions & flushes 2024-11-11T13:36:00,288 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:36:00,288 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:36:00,288 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. after waiting 0 ms 2024-11-11T13:36:00,288 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:36:00,288 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing e913da51831f130352321c03b81d22d5 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T13:36:00,299 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111101309726ffc443db757cde577136bf0_e913da51831f130352321c03b81d22d5 is 50, key is test_row_0/A:col10/1731332160119/Put/seqid=0 2024-11-11T13:36:00,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742046_1222 (size=12454) 2024-11-11T13:36:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T13:36:00,704 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:00,710 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111101309726ffc443db757cde577136bf0_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111101309726ffc443db757cde577136bf0_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:00,711 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/b512eabda2104a2ba2fcb0a7d9b16060, store: [table=TestAcidGuarantees family=A region=e913da51831f130352321c03b81d22d5] 2024-11-11T13:36:00,712 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/b512eabda2104a2ba2fcb0a7d9b16060 is 175, key is test_row_0/A:col10/1731332160119/Put/seqid=0 2024-11-11T13:36:00,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742047_1223 (size=31255) 2024-11-11T13:36:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T13:36:01,117 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/b512eabda2104a2ba2fcb0a7d9b16060 2024-11-11T13:36:01,124 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/e212f27bd2d843eaab3776b444d1d987 is 50, key is test_row_0/B:col10/1731332160119/Put/seqid=0 2024-11-11T13:36:01,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742048_1224 (size=12301) 2024-11-11T13:36:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T13:36:01,529 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/e212f27bd2d843eaab3776b444d1d987 2024-11-11T13:36:01,540 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/1c3b5af79c4449558f7095b2ed14cd75 is 50, key is test_row_0/C:col10/1731332160119/Put/seqid=0 2024-11-11T13:36:01,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742049_1225 (size=12301) 2024-11-11T13:36:01,945 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/1c3b5af79c4449558f7095b2ed14cd75 2024-11-11T13:36:01,950 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/A/b512eabda2104a2ba2fcb0a7d9b16060 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/b512eabda2104a2ba2fcb0a7d9b16060 2024-11-11T13:36:01,954 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/b512eabda2104a2ba2fcb0a7d9b16060, entries=150, sequenceid=340, filesize=30.5 K 2024-11-11T13:36:01,954 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/B/e212f27bd2d843eaab3776b444d1d987 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e212f27bd2d843eaab3776b444d1d987 2024-11-11T13:36:01,959 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e212f27bd2d843eaab3776b444d1d987, entries=150, sequenceid=340, filesize=12.0 K 2024-11-11T13:36:01,960 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/.tmp/C/1c3b5af79c4449558f7095b2ed14cd75 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/1c3b5af79c4449558f7095b2ed14cd75 2024-11-11T13:36:01,965 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/1c3b5af79c4449558f7095b2ed14cd75, entries=150, sequenceid=340, filesize=12.0 K 2024-11-11T13:36:01,966 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e913da51831f130352321c03b81d22d5 in 1678ms, sequenceid=340, compaction requested=true 2024-11-11T13:36:01,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/80abe52aa2fa4eb2b3ac841278709ffe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a2384366e23e4262bd29dbebdf841efd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c8f3cc36735947e1a304cb6b59d42fe4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/245c2792e1944ca6a0999123c742d624, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/495fd2ab746b448abbfb64b606863a6f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/8c0ad0f4f68e4891b0964b0c28e4ea08, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/29c3be909230453aa921240f6174f237, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/272f0a868c9c463a80c4269842243755, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/4b7e70fc190041c785c7b8976acb4713, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/df176d1b5bf44963a41878a5fd02f25a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9044d5f581b84a34b1b04e611b0e0718, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/ec144db0b5ac455b9fde69e4fe819917, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/3c6ffa3cf9c447d79052db1bee583318, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/b014dc67ab57443b8e9a63dcbbeda0a7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a7f9bef18d294a4a8acd38e8fd96d58b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a39dd93c92ae4e1a94d288ba114e40ca, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/da83647d7a8644ccbb3f4627b576ea35, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/02af8222a8f346bf9c51dc2d3656f9a1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/d5f6b335bf13471e957ec7e6f4c15651, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9877b3e336634f17aaa417898729b3bd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c24ef28bac81461280ad32628e9a7444] to archive 2024-11-11T13:36:01,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:36:01,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/80abe52aa2fa4eb2b3ac841278709ffe to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/80abe52aa2fa4eb2b3ac841278709ffe 2024-11-11T13:36:01,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a2384366e23e4262bd29dbebdf841efd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a2384366e23e4262bd29dbebdf841efd 2024-11-11T13:36:01,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c8f3cc36735947e1a304cb6b59d42fe4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c8f3cc36735947e1a304cb6b59d42fe4 2024-11-11T13:36:01,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/245c2792e1944ca6a0999123c742d624 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/245c2792e1944ca6a0999123c742d624 2024-11-11T13:36:01,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/495fd2ab746b448abbfb64b606863a6f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/495fd2ab746b448abbfb64b606863a6f 2024-11-11T13:36:01,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/8c0ad0f4f68e4891b0964b0c28e4ea08 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/8c0ad0f4f68e4891b0964b0c28e4ea08 2024-11-11T13:36:01,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/29c3be909230453aa921240f6174f237 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/29c3be909230453aa921240f6174f237 2024-11-11T13:36:01,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/272f0a868c9c463a80c4269842243755 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/272f0a868c9c463a80c4269842243755 2024-11-11T13:36:01,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/4b7e70fc190041c785c7b8976acb4713 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/4b7e70fc190041c785c7b8976acb4713 2024-11-11T13:36:01,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/df176d1b5bf44963a41878a5fd02f25a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/df176d1b5bf44963a41878a5fd02f25a 2024-11-11T13:36:01,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9044d5f581b84a34b1b04e611b0e0718 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9044d5f581b84a34b1b04e611b0e0718 2024-11-11T13:36:01,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/ec144db0b5ac455b9fde69e4fe819917 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/ec144db0b5ac455b9fde69e4fe819917 2024-11-11T13:36:01,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/3c6ffa3cf9c447d79052db1bee583318 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/3c6ffa3cf9c447d79052db1bee583318 2024-11-11T13:36:01,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/b014dc67ab57443b8e9a63dcbbeda0a7 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/b014dc67ab57443b8e9a63dcbbeda0a7 2024-11-11T13:36:01,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a7f9bef18d294a4a8acd38e8fd96d58b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a7f9bef18d294a4a8acd38e8fd96d58b 2024-11-11T13:36:01,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a39dd93c92ae4e1a94d288ba114e40ca to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a39dd93c92ae4e1a94d288ba114e40ca 2024-11-11T13:36:01,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/da83647d7a8644ccbb3f4627b576ea35 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/da83647d7a8644ccbb3f4627b576ea35 2024-11-11T13:36:01,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/02af8222a8f346bf9c51dc2d3656f9a1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/02af8222a8f346bf9c51dc2d3656f9a1 2024-11-11T13:36:01,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/d5f6b335bf13471e957ec7e6f4c15651 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/d5f6b335bf13471e957ec7e6f4c15651 2024-11-11T13:36:01,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9877b3e336634f17aaa417898729b3bd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/9877b3e336634f17aaa417898729b3bd 2024-11-11T13:36:02,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c24ef28bac81461280ad32628e9a7444 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/c24ef28bac81461280ad32628e9a7444 2024-11-11T13:36:02,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/18a2cd82f0b843efbd1b09317e25bae4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/a1343df06f8e4a578405cce9347dd7c5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/2bd528faf1f747c1a12ad9a52a2a8c85, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b5d253a368d945a9ab6e73c13ece505b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1e1ebaf4c4f942498d571537dc6cafc7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/c567fe0b1da3469f90e9df9fabc78c20, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1c4d0b99e2f1496db6bbf983516ee336, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/20b3c772763046e3b420520fc59e264c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e5ac53a722c64cd38cbc2e2f0b97bc0c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/283099fcfca94107b21d4430bc64a589, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e3dc97c611f34e33843894c430d397c9, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1ffd5d87f72148ad86e489b077c7e40d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/87ded0e8a41b4a559c139f60ed7334cb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/4c71c5c2eba04b2cbfe4ee047a455412, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/79c815300e1f4841aadb5fb613d03208, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/0b25f69cfb4b47dd8da10c05fd923a37, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d9127ee163de4a7fae6560a053794f56, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/eac21cc5c38b4fbab8c6c02468a079b1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b338d75cfe13483a80ba3d229adfa80c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d7896eee9e9740b889ddccc48aac3cda, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/54bf7425de9d42769b462efe3926c2b4] to archive 2024-11-11T13:36:02,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:36:02,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/18a2cd82f0b843efbd1b09317e25bae4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/18a2cd82f0b843efbd1b09317e25bae4 2024-11-11T13:36:02,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/a1343df06f8e4a578405cce9347dd7c5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/a1343df06f8e4a578405cce9347dd7c5 2024-11-11T13:36:02,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/2bd528faf1f747c1a12ad9a52a2a8c85 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/2bd528faf1f747c1a12ad9a52a2a8c85 2024-11-11T13:36:02,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b5d253a368d945a9ab6e73c13ece505b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b5d253a368d945a9ab6e73c13ece505b 2024-11-11T13:36:02,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1e1ebaf4c4f942498d571537dc6cafc7 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1e1ebaf4c4f942498d571537dc6cafc7 2024-11-11T13:36:02,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/c567fe0b1da3469f90e9df9fabc78c20 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/c567fe0b1da3469f90e9df9fabc78c20 2024-11-11T13:36:02,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1c4d0b99e2f1496db6bbf983516ee336 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1c4d0b99e2f1496db6bbf983516ee336 2024-11-11T13:36:02,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/20b3c772763046e3b420520fc59e264c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/20b3c772763046e3b420520fc59e264c 2024-11-11T13:36:02,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e5ac53a722c64cd38cbc2e2f0b97bc0c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e5ac53a722c64cd38cbc2e2f0b97bc0c 2024-11-11T13:36:02,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/283099fcfca94107b21d4430bc64a589 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/283099fcfca94107b21d4430bc64a589 2024-11-11T13:36:02,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e3dc97c611f34e33843894c430d397c9 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e3dc97c611f34e33843894c430d397c9 2024-11-11T13:36:02,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1ffd5d87f72148ad86e489b077c7e40d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/1ffd5d87f72148ad86e489b077c7e40d 2024-11-11T13:36:02,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/87ded0e8a41b4a559c139f60ed7334cb to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/87ded0e8a41b4a559c139f60ed7334cb 2024-11-11T13:36:02,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/4c71c5c2eba04b2cbfe4ee047a455412 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/4c71c5c2eba04b2cbfe4ee047a455412 2024-11-11T13:36:02,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/79c815300e1f4841aadb5fb613d03208 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/79c815300e1f4841aadb5fb613d03208 2024-11-11T13:36:02,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/0b25f69cfb4b47dd8da10c05fd923a37 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/0b25f69cfb4b47dd8da10c05fd923a37 2024-11-11T13:36:02,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d9127ee163de4a7fae6560a053794f56 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d9127ee163de4a7fae6560a053794f56 2024-11-11T13:36:02,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/eac21cc5c38b4fbab8c6c02468a079b1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/eac21cc5c38b4fbab8c6c02468a079b1 2024-11-11T13:36:02,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b338d75cfe13483a80ba3d229adfa80c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/b338d75cfe13483a80ba3d229adfa80c 2024-11-11T13:36:02,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d7896eee9e9740b889ddccc48aac3cda to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/d7896eee9e9740b889ddccc48aac3cda 2024-11-11T13:36:02,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/54bf7425de9d42769b462efe3926c2b4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/54bf7425de9d42769b462efe3926c2b4 2024-11-11T13:36:02,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/5e0e86423c27462a8089672d26e3137a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/86f0867589bf4104ac8348ed0f4aa3db, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/04980b51d3d94715bbfb174a1c4ddbcd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/250e367a70ee494793141298da4b0d70, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/787f597937564fadbbc0b148676e4430, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b4ca82302248e5ba6cfaf3eb6a184b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/3264c56f1eb44ea6b9ad06a475312f41, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/bfaff851454d428e932486af0b7dcf02, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/4b0876a3161a43b8a3a75431e07357d6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/b759ba23e2f74ac88e6f16422755c8ff, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/2672dd6f14214e10a2c9e8231b1a9b70, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/939db6ad4c604bee8f7ba71ad94853e1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/61a61bc555944cae8cb399969f8e1857, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/8cc38a2528c3437290ec714ca73fdbba, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/56658159702f435bb804390e593d73bf, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/97fbc54fb5784709ad9475642aa9a613, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b0d374493b49e89017f282d07a6989, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/30a0099b875a448988625c776fbcabcc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/a8b18df2ab144a289c6603a9c6c3c19d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/3e7dd916060b4bab80f61ea72dde01a8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d7e97e34adcc4cd6a47cc39a64fc64fd] to archive 2024-11-11T13:36:02,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:36:02,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/5e0e86423c27462a8089672d26e3137a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/5e0e86423c27462a8089672d26e3137a 2024-11-11T13:36:02,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/86f0867589bf4104ac8348ed0f4aa3db to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/86f0867589bf4104ac8348ed0f4aa3db 2024-11-11T13:36:02,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/04980b51d3d94715bbfb174a1c4ddbcd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/04980b51d3d94715bbfb174a1c4ddbcd 2024-11-11T13:36:02,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/250e367a70ee494793141298da4b0d70 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/250e367a70ee494793141298da4b0d70 2024-11-11T13:36:02,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/787f597937564fadbbc0b148676e4430 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/787f597937564fadbbc0b148676e4430 2024-11-11T13:36:02,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b4ca82302248e5ba6cfaf3eb6a184b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b4ca82302248e5ba6cfaf3eb6a184b 2024-11-11T13:36:02,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/3264c56f1eb44ea6b9ad06a475312f41 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/3264c56f1eb44ea6b9ad06a475312f41 2024-11-11T13:36:02,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/bfaff851454d428e932486af0b7dcf02 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/bfaff851454d428e932486af0b7dcf02 2024-11-11T13:36:02,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/4b0876a3161a43b8a3a75431e07357d6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/4b0876a3161a43b8a3a75431e07357d6 2024-11-11T13:36:02,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/b759ba23e2f74ac88e6f16422755c8ff to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/b759ba23e2f74ac88e6f16422755c8ff 2024-11-11T13:36:02,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/2672dd6f14214e10a2c9e8231b1a9b70 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/2672dd6f14214e10a2c9e8231b1a9b70 2024-11-11T13:36:02,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/939db6ad4c604bee8f7ba71ad94853e1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/939db6ad4c604bee8f7ba71ad94853e1 2024-11-11T13:36:02,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/61a61bc555944cae8cb399969f8e1857 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/61a61bc555944cae8cb399969f8e1857 2024-11-11T13:36:02,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/8cc38a2528c3437290ec714ca73fdbba to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/8cc38a2528c3437290ec714ca73fdbba 2024-11-11T13:36:02,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/56658159702f435bb804390e593d73bf to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/56658159702f435bb804390e593d73bf 2024-11-11T13:36:02,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/97fbc54fb5784709ad9475642aa9a613 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/97fbc54fb5784709ad9475642aa9a613 2024-11-11T13:36:02,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b0d374493b49e89017f282d07a6989 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d3b0d374493b49e89017f282d07a6989 2024-11-11T13:36:02,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/30a0099b875a448988625c776fbcabcc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/30a0099b875a448988625c776fbcabcc 2024-11-11T13:36:02,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/a8b18df2ab144a289c6603a9c6c3c19d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/a8b18df2ab144a289c6603a9c6c3c19d 2024-11-11T13:36:02,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/3e7dd916060b4bab80f61ea72dde01a8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/3e7dd916060b4bab80f61ea72dde01a8 2024-11-11T13:36:02,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d7e97e34adcc4cd6a47cc39a64fc64fd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/d7e97e34adcc4cd6a47cc39a64fc64fd 2024-11-11T13:36:02,073 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/recovered.edits/343.seqid, newMaxSeqId=343, maxSeqId=4 2024-11-11T13:36:02,076 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. 2024-11-11T13:36:02,076 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for e913da51831f130352321c03b81d22d5: Waiting for close lock at 1731332160288Running coprocessor pre-close hooks at 1731332160288Disabling compacts and flushes for region at 1731332160288Disabling writes for close at 1731332160288Obtaining lock to block concurrent updates at 1731332160288Preparing flush snapshotting stores in e913da51831f130352321c03b81d22d5 at 1731332160288Finished memstore snapshotting TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731332160288Flushing stores of TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5. at 1731332160289 (+1 ms)Flushing e913da51831f130352321c03b81d22d5/A: creating writer at 1731332160289Flushing mob file e913da51831f130352321c03b81d22d5/A: appending metadata at 1731332160299 (+10 ms)Flushing mob file e913da51831f130352321c03b81d22d5/A: closing flushed file at 1731332160299Flushing e913da51831f130352321c03b81d22d5/A: appending metadata at 1731332160711 (+412 ms)Flushing e913da51831f130352321c03b81d22d5/A: closing flushed file at 1731332160712 (+1 ms)Flushing e913da51831f130352321c03b81d22d5/B: creating writer at 1731332161122 (+410 ms)Flushing e913da51831f130352321c03b81d22d5/B: appending metadata at 1731332161124 (+2 ms)Flushing e913da51831f130352321c03b81d22d5/B: closing flushed file at 1731332161124Flushing e913da51831f130352321c03b81d22d5/C: creating writer at 1731332161535 (+411 ms)Flushing e913da51831f130352321c03b81d22d5/C: appending metadata at 1731332161539 (+4 ms)Flushing e913da51831f130352321c03b81d22d5/C: closing flushed file at 1731332161539Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26b7a9d7: reopening flushed file at 1731332161949 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2285c408: reopening flushed file at 1731332161954 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4746d6a7: reopening flushed file at 1731332161959 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e913da51831f130352321c03b81d22d5 in 1678ms, sequenceid=340, compaction requested=true at 1731332161966 (+7 ms)Writing region close event to WAL at 1731332162069 (+103 ms)Running coprocessor post-close hooks at 1731332162076 (+7 ms)Closed at 1731332162076 2024-11-11T13:36:02,078 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,078 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=e913da51831f130352321c03b81d22d5, regionState=CLOSED 2024-11-11T13:36:02,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:36:02,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T13:36:02,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure e913da51831f130352321c03b81d22d5, server=37d26ecf6460,36877,1731332101576 in 1.9450 sec 2024-11-11T13:36:02,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-11T13:36:02,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e913da51831f130352321c03b81d22d5, UNASSIGN in 1.9490 sec 2024-11-11T13:36:02,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T13:36:02,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9520 sec 2024-11-11T13:36:02,088 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332162087"}]},"ts":"1731332162087"} 2024-11-11T13:36:02,089 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T13:36:02,089 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T13:36:02,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9670 sec 2024-11-11T13:36:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T13:36:02,270 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:02,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T13:36:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:02,272 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T13:36:02,273 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:02,275 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,277 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/recovered.edits] 2024-11-11T13:36:02,280 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/113d4567e5b54f50963c3a87a0687130 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/113d4567e5b54f50963c3a87a0687130 2024-11-11T13:36:02,281 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a8311f6b3272496daf8cf0e1291e3b55 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/a8311f6b3272496daf8cf0e1291e3b55 2024-11-11T13:36:02,282 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/b512eabda2104a2ba2fcb0a7d9b16060 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/A/b512eabda2104a2ba2fcb0a7d9b16060 2024-11-11T13:36:02,284 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/6c7680d89ccf450ab6c45863f17168c7 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/6c7680d89ccf450ab6c45863f17168c7 2024-11-11T13:36:02,285 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/c2c73c49a0a54c8b8314ed3d7243577f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/c2c73c49a0a54c8b8314ed3d7243577f 2024-11-11T13:36:02,286 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e212f27bd2d843eaab3776b444d1d987 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/B/e212f27bd2d843eaab3776b444d1d987 2024-11-11T13:36:02,288 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/03b05f35fc4d4edea91fdaf2e7d18331 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/03b05f35fc4d4edea91fdaf2e7d18331 2024-11-11T13:36:02,289 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/0d15461805944115a3c2324183d4d002 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/0d15461805944115a3c2324183d4d002 2024-11-11T13:36:02,290 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/1c3b5af79c4449558f7095b2ed14cd75 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/C/1c3b5af79c4449558f7095b2ed14cd75 2024-11-11T13:36:02,292 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/recovered.edits/343.seqid to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5/recovered.edits/343.seqid 2024-11-11T13:36:02,292 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,292 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T13:36:02,293 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T13:36:02,293 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T13:36:02,296 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111104eac30d20e34b639e7fdc85798f3373_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111104eac30d20e34b639e7fdc85798f3373_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,297 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111096ad2fdd5054e76a105c4e0ec8efca6_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111096ad2fdd5054e76a105c4e0ec8efca6_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,298 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110cbdfccbe6fb4f518ef316688b7d77ec_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110cbdfccbe6fb4f518ef316688b7d77ec_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,299 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111101309726ffc443db757cde577136bf0_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111101309726ffc443db757cde577136bf0_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,299 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111e04e1ccf56c4f3ba07fb1ff195b27ff_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111e04e1ccf56c4f3ba07fb1ff195b27ff_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,300 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111120fd7efde95b49178565560a84a84506_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111120fd7efde95b49178565560a84a84506_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,301 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111427d66d5deda4750bc409a06ff3fc51f_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111427d66d5deda4750bc409a06ff3fc51f_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,302 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111149385cc65bb3442eb8e9f6332473eb75_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111149385cc65bb3442eb8e9f6332473eb75_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,303 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114cf321a579034b7886a15bcb844489d0_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114cf321a579034b7886a15bcb844489d0_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,305 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116c3409dcd3424118a3ea7d0f3490d2b3_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116c3409dcd3424118a3ea7d0f3490d2b3_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,306 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116eb37c41911b4cd19bfd100efadef1e1_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116eb37c41911b4cd19bfd100efadef1e1_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,307 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111179d61fe79a054657af5b0072a9b38a00_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111179d61fe79a054657af5b0072a9b38a00_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,308 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a4f169f5fa6b4464a0435c0c1f8b6159_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a4f169f5fa6b4464a0435c0c1f8b6159_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,310 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d7dc2b55bd89427a8bdcd8460bfa823c_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d7dc2b55bd89427a8bdcd8460bfa823c_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,311 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e18f8eb66277424e91d4a54434df3a37_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e18f8eb66277424e91d4a54434df3a37_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,312 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e337f550711740eb90b6a2aef6fae44d_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e337f550711740eb90b6a2aef6fae44d_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,314 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fa69bce6e98b4a738c325fc1b96cc9b2_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fa69bce6e98b4a738c325fc1b96cc9b2_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,315 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fe0e2129ca1b4bc593f1a294552fa355_e913da51831f130352321c03b81d22d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fe0e2129ca1b4bc593f1a294552fa355_e913da51831f130352321c03b81d22d5 2024-11-11T13:36:02,316 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T13:36:02,321 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:02,323 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T13:36:02,326 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T13:36:02,328 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:02,328 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T13:36:02,328 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731332162328"}]},"ts":"9223372036854775807"} 2024-11-11T13:36:02,330 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T13:36:02,330 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e913da51831f130352321c03b81d22d5, NAME => 'TestAcidGuarantees,,1731332133324.e913da51831f130352321c03b81d22d5.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T13:36:02,331 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T13:36:02,331 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731332162331"}]},"ts":"9223372036854775807"} 2024-11-11T13:36:02,333 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T13:36:02,334 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:02,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-11T13:36:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T13:36:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T13:36:02,380 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:02,391 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239 (was 239), OpenFileDescriptor=455 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=499 (was 490) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2931 (was 3099) 2024-11-11T13:36:02,401 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=499, ProcessCount=11, AvailableMemoryMB=2929 2024-11-11T13:36:02,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T13:36:02,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:36:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:02,406 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T13:36:02,406 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:02,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-11T13:36:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T13:36:02,408 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T13:36:02,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742050_1226 (size=959) 2024-11-11T13:36:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T13:36:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T13:36:02,821 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92 2024-11-11T13:36:02,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742051_1227 (size=53) 2024-11-11T13:36:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T13:36:03,229 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:36:03,229 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7cb6411e18fb1479182e4865079a3fa6, disabling compactions & flushes 2024-11-11T13:36:03,229 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:03,229 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:03,229 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. after waiting 0 ms 2024-11-11T13:36:03,229 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:03,229 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:03,229 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7cb6411e18fb1479182e4865079a3fa6: Waiting for close lock at 1731332163229Disabling compacts and flushes for region at 1731332163229Disabling writes for close at 1731332163229Writing region close event to WAL at 1731332163229Closed at 1731332163229 2024-11-11T13:36:03,230 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T13:36:03,230 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731332163230"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731332163230"}]},"ts":"1731332163230"} 2024-11-11T13:36:03,232 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T13:36:03,232 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T13:36:03,233 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332163232"}]},"ts":"1731332163232"} 2024-11-11T13:36:03,234 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T13:36:03,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cb6411e18fb1479182e4865079a3fa6, ASSIGN}] 2024-11-11T13:36:03,236 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cb6411e18fb1479182e4865079a3fa6, ASSIGN 2024-11-11T13:36:03,236 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cb6411e18fb1479182e4865079a3fa6, ASSIGN; state=OFFLINE, location=37d26ecf6460,36877,1731332101576; forceNewPlan=false, retain=false 2024-11-11T13:36:03,387 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=7cb6411e18fb1479182e4865079a3fa6, regionState=OPENING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:03,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cb6411e18fb1479182e4865079a3fa6, ASSIGN because future has completed 2024-11-11T13:36:03,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:36:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T13:36:03,549 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:03,549 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:36:03,549 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,550 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:36:03,550 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,550 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,552 INFO [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,554 INFO [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cb6411e18fb1479182e4865079a3fa6 columnFamilyName A 2024-11-11T13:36:03,555 DEBUG [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:03,555 INFO [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] regionserver.HStore(327): Store=7cb6411e18fb1479182e4865079a3fa6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:36:03,555 INFO [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,557 INFO [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cb6411e18fb1479182e4865079a3fa6 columnFamilyName B 2024-11-11T13:36:03,557 DEBUG [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:03,558 INFO [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] regionserver.HStore(327): Store=7cb6411e18fb1479182e4865079a3fa6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:36:03,558 INFO [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,560 INFO [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cb6411e18fb1479182e4865079a3fa6 columnFamilyName C 2024-11-11T13:36:03,560 DEBUG [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:03,561 INFO [StoreOpener-7cb6411e18fb1479182e4865079a3fa6-1 {}] regionserver.HStore(327): Store=7cb6411e18fb1479182e4865079a3fa6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:36:03,561 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,562 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,563 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,564 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,564 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,565 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T13:36:03,567 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,569 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:36:03,570 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 7cb6411e18fb1479182e4865079a3fa6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59016566, jitterRate=-0.120584636926651}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T13:36:03,570 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:03,571 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 7cb6411e18fb1479182e4865079a3fa6: Running coprocessor pre-open hook at 1731332163550Writing region info on filesystem at 1731332163550Initializing all the Stores at 1731332163552 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332163552Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332163552Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332163552Cleaning up temporary data from old regions at 1731332163564 (+12 ms)Running coprocessor post-open hooks at 1731332163570 (+6 ms)Region opened successfully at 1731332163571 (+1 ms) 2024-11-11T13:36:03,572 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., pid=59, masterSystemTime=1731332163542 2024-11-11T13:36:03,574 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:03,574 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:03,574 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=7cb6411e18fb1479182e4865079a3fa6, regionState=OPEN, openSeqNum=2, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:03,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:36:03,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T13:36:03,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 in 187 msec 2024-11-11T13:36:03,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-11T13:36:03,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cb6411e18fb1479182e4865079a3fa6, ASSIGN in 346 msec 2024-11-11T13:36:03,583 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T13:36:03,583 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332163583"}]},"ts":"1731332163583"} 2024-11-11T13:36:03,585 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T13:36:03,586 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T13:36:03,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-11T13:36:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T13:36:04,550 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:04,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447ea5bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:04,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:04,552 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56269cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:04,554 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:04,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca66ed8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,555 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:04,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:04,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:04,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:04,557 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T13:36:04,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 37d26ecf6460,35609,1731332100707 2024-11-11T13:36:04,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2191a235 2024-11-11T13:36:04,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T13:36:04,559 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T13:36:04,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b313f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:04,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:04,562 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441c7907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:04,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:04,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:04,564 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:04,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58ccfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a21a52a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:04,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:04,566 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:04,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:04,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:04,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37490b3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:04,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:04,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:04,567 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:04,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef080b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@931de11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:04,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:04,569 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6934c34f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:04,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:04,570 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:04,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c4637b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2904c2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:04,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:04,572 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402ea618, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:04,573 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:04,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e8fcba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23477c32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:04,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:04,575 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32062301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:04,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:04,576 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:04,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e82933, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7315f311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:04,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:04,578 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283319d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:04,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:04,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:04,579 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:04,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6689c259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57285f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:04,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:04,581 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c60dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:04,583 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:04,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ac3376, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca57e3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:04,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:04,584 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0c5f70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:04,586 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:04,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a3dd81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@848e0ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,587 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:04,587 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:04,587 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:04,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:04,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd6fbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:04,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7141904a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fcf6cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:04,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:04,595 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6360192b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:04,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:04,596 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1facbcc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:04,597 DEBUG [Thread-1080 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:04,597 DEBUG [Thread-1082 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:04,598 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:04,598 DEBUG [Thread-1073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:04,598 DEBUG [Thread-1078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:04,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:04,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:04,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:04,599 DEBUG [Thread-1075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:04,599 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:04,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:04,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:04,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:04,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:04,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:04,602 DEBUG [Thread-1067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:36:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:36:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:36:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:04,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:36:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:04,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:36:04,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:36:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:36:04,607 DEBUG [Thread-1086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:04,607 DEBUG [Thread-1084 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:04,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:36:04,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:04,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-11T13:36:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T13:36:04,612 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:04,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:04,613 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:04,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:36:04,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:36:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:04,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T13:36:04,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/a8dea4d5ff8542fb891ea6d805ead2e7 is 50, key is test_row_0/A:col10/1731332164613/Put/seqid=0 2024-11-11T13:36:04,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742052_1228 (size=12001) 2024-11-11T13:36:04,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/a8dea4d5ff8542fb891ea6d805ead2e7 2024-11-11T13:36:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T13:36:04,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332224675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:04,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332224720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:04,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332224724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:04,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332224726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:04,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/2ac919496308476b9afe93930d185c7d is 50, key is test_row_0/B:col10/1731332164613/Put/seqid=0 2024-11-11T13:36:04,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332224725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:04,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:04,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:04,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:04,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:04,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:04,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:04,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742053_1229 (size=12001) 2024-11-11T13:36:04,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/2ac919496308476b9afe93930d185c7d 2024-11-11T13:36:04,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1baf5c8e9b684c448f58be9785b3018a is 50, key is test_row_0/C:col10/1731332164613/Put/seqid=0 2024-11-11T13:36:04,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742054_1230 (size=12001) 2024-11-11T13:36:04,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:04,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332224829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:04,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:04,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:04,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332224843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:04,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332224844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:04,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:04,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332224844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:04,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332224844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:04,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:04,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:04,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:04,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:04,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:04,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:04,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:04,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T13:36:05,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332225039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332225049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332225049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332225049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332225055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:05,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:05,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1baf5c8e9b684c448f58be9785b3018a 2024-11-11T13:36:05,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/a8dea4d5ff8542fb891ea6d805ead2e7 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/a8dea4d5ff8542fb891ea6d805ead2e7 2024-11-11T13:36:05,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/a8dea4d5ff8542fb891ea6d805ead2e7, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T13:36:05,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/2ac919496308476b9afe93930d185c7d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/2ac919496308476b9afe93930d185c7d 2024-11-11T13:36:05,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/2ac919496308476b9afe93930d185c7d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T13:36:05,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1baf5c8e9b684c448f58be9785b3018a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1baf5c8e9b684c448f58be9785b3018a 2024-11-11T13:36:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T13:36:05,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:05,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:05,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1baf5c8e9b684c448f58be9785b3018a, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T13:36:05,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for 7cb6411e18fb1479182e4865079a3fa6 in 633ms, sequenceid=15, compaction requested=false 2024-11-11T13:36:05,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:05,350 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:05,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:36:05,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/7092ed912da04329bb109412e197899a is 50, key is test_row_0/A:col10/1731332164723/Put/seqid=0 2024-11-11T13:36:05,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742055_1231 (size=12001) 2024-11-11T13:36:05,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332225364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332225365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332225368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332225369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332225369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:05,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:05,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332225480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332225480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332225480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332225480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:05,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:05,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332225689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332225690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332225690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332225691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:05,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:05,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:05,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:05,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T13:36:05,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/7092ed912da04329bb109412e197899a 2024-11-11T13:36:05,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/9cc8648708b74a199d0cee2c2ae13727 is 50, key is test_row_0/B:col10/1731332164723/Put/seqid=0 2024-11-11T13:36:05,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742056_1232 (size=12001) 2024-11-11T13:36:05,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/9cc8648708b74a199d0cee2c2ae13727 2024-11-11T13:36:05,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/0cfc47a2ae674b8c8dd700e9acd6bc64 is 50, key is test_row_0/C:col10/1731332164723/Put/seqid=0 2024-11-11T13:36:05,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742057_1233 (size=12001) 2024-11-11T13:36:05,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/0cfc47a2ae674b8c8dd700e9acd6bc64 2024-11-11T13:36:05,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/7092ed912da04329bb109412e197899a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7092ed912da04329bb109412e197899a 2024-11-11T13:36:05,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:05,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:05,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:05,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:05,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7092ed912da04329bb109412e197899a, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T13:36:05,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/9cc8648708b74a199d0cee2c2ae13727 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/9cc8648708b74a199d0cee2c2ae13727 2024-11-11T13:36:05,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/9cc8648708b74a199d0cee2c2ae13727, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T13:36:05,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/0cfc47a2ae674b8c8dd700e9acd6bc64 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/0cfc47a2ae674b8c8dd700e9acd6bc64 2024-11-11T13:36:05,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/0cfc47a2ae674b8c8dd700e9acd6bc64, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T13:36:05,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7cb6411e18fb1479182e4865079a3fa6 in 525ms, sequenceid=40, compaction requested=false 2024-11-11T13:36:05,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:05,881 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:05,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T13:36:05,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/7ca56fc8d3d34c08b1b6933a417b6323 is 50, key is test_row_0/A:col10/1731332165361/Put/seqid=0 2024-11-11T13:36:05,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742058_1234 (size=12001) 2024-11-11T13:36:05,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/7ca56fc8d3d34c08b1b6933a417b6323 2024-11-11T13:36:05,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/42b69ab86e124cd3bfb89318d8897bec is 50, key is test_row_0/B:col10/1731332165361/Put/seqid=0 2024-11-11T13:36:05,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742059_1235 (size=12001) 2024-11-11T13:36:05,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/42b69ab86e124cd3bfb89318d8897bec 2024-11-11T13:36:06,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1e9ffd84803d471da3e430718901cc60 is 50, key is test_row_0/C:col10/1731332165361/Put/seqid=0 2024-11-11T13:36:06,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332226004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332226021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332226022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332226022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332226023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742060_1236 (size=12001) 2024-11-11T13:36:06,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1e9ffd84803d471da3e430718901cc60 2024-11-11T13:36:06,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/7ca56fc8d3d34c08b1b6933a417b6323 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7ca56fc8d3d34c08b1b6933a417b6323 2024-11-11T13:36:06,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7ca56fc8d3d34c08b1b6933a417b6323, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T13:36:06,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/42b69ab86e124cd3bfb89318d8897bec as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/42b69ab86e124cd3bfb89318d8897bec 2024-11-11T13:36:06,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/42b69ab86e124cd3bfb89318d8897bec, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T13:36:06,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1e9ffd84803d471da3e430718901cc60 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1e9ffd84803d471da3e430718901cc60 2024-11-11T13:36:06,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332226131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1e9ffd84803d471da3e430718901cc60, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T13:36:06,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7cb6411e18fb1479182e4865079a3fa6 in 262ms, sequenceid=52, compaction requested=true 2024-11-11T13:36:06,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:06,144 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:06,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:06,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:06,144 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:06,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:06,151 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:06,151 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/B is initiating minor compaction (all files) 2024-11-11T13:36:06,151 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/B in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,151 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/2ac919496308476b9afe93930d185c7d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/9cc8648708b74a199d0cee2c2ae13727, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/42b69ab86e124cd3bfb89318d8897bec] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=35.2 K 2024-11-11T13:36:06,151 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:06,151 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/A is initiating minor compaction (all files) 2024-11-11T13:36:06,151 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/A in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,152 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/a8dea4d5ff8542fb891ea6d805ead2e7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7092ed912da04329bb109412e197899a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7ca56fc8d3d34c08b1b6933a417b6323] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=35.2 K 2024-11-11T13:36:06,152 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:06,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T13:36:06,154 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8dea4d5ff8542fb891ea6d805ead2e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731332164609 2024-11-11T13:36:06,154 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ac919496308476b9afe93930d185c7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731332164609 2024-11-11T13:36:06,158 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cc8648708b74a199d0cee2c2ae13727, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731332164674 2024-11-11T13:36:06,158 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 42b69ab86e124cd3bfb89318d8897bec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332165361 2024-11-11T13:36:06,161 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7092ed912da04329bb109412e197899a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731332164674 2024-11-11T13:36:06,169 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ca56fc8d3d34c08b1b6933a417b6323, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332165361 2024-11-11T13:36:06,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332226167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332226169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332226171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332226172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/5336facef3a647838f8b7357e364ec4a is 50, key is test_row_0/A:col10/1731332166021/Put/seqid=0 2024-11-11T13:36:06,188 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#A#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:06,188 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/a48044f21e35401fb1752ad39fe98660 is 50, key is test_row_0/A:col10/1731332165361/Put/seqid=0 2024-11-11T13:36:06,209 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#B#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:06,210 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/f9f4a86ee2b34d27801735dfa1ae8a0d is 50, key is test_row_0/B:col10/1731332165361/Put/seqid=0 2024-11-11T13:36:06,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742061_1237 (size=14341) 2024-11-11T13:36:06,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332226280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332226280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742062_1238 (size=12104) 2024-11-11T13:36:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332226292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332226295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,307 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/a48044f21e35401fb1752ad39fe98660 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/a48044f21e35401fb1752ad39fe98660 2024-11-11T13:36:06,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742063_1239 (size=12104) 2024-11-11T13:36:06,323 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/A of 7cb6411e18fb1479182e4865079a3fa6 into a48044f21e35401fb1752ad39fe98660(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:06,323 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:06,323 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/A, priority=13, startTime=1731332166144; duration=0sec 2024-11-11T13:36:06,323 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:06,323 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:A 2024-11-11T13:36:06,324 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:06,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:06,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:06,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,329 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:06,330 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/C is initiating minor compaction (all files) 2024-11-11T13:36:06,330 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/C in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,330 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1baf5c8e9b684c448f58be9785b3018a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/0cfc47a2ae674b8c8dd700e9acd6bc64, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1e9ffd84803d471da3e430718901cc60] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=35.2 K 2024-11-11T13:36:06,330 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1baf5c8e9b684c448f58be9785b3018a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731332164609 2024-11-11T13:36:06,331 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cfc47a2ae674b8c8dd700e9acd6bc64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731332164674 2024-11-11T13:36:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,334 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e9ffd84803d471da3e430718901cc60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332165361 2024-11-11T13:36:06,336 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/f9f4a86ee2b34d27801735dfa1ae8a0d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f9f4a86ee2b34d27801735dfa1ae8a0d 2024-11-11T13:36:06,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,345 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/B of 7cb6411e18fb1479182e4865079a3fa6 into f9f4a86ee2b34d27801735dfa1ae8a0d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:06,345 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332226339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,345 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/B, priority=13, startTime=1731332166144; duration=0sec 2024-11-11T13:36:06,345 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:06,345 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:B 2024-11-11T13:36:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,349 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#C#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:06,350 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/d0cd7bac37c64437aadbce78c458ee64 is 50, key is test_row_0/C:col10/1731332165361/Put/seqid=0 2024-11-11T13:36:06,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742064_1240 (size=12104) 2024-11-11T13:36:06,377 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/d0cd7bac37c64437aadbce78c458ee64 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d0cd7bac37c64437aadbce78c458ee64 2024-11-11T13:36:06,387 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/C of 7cb6411e18fb1479182e4865079a3fa6 into d0cd7bac37c64437aadbce78c458ee64(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:06,387 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:06,387 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/C, priority=13, startTime=1731332166149; duration=0sec 2024-11-11T13:36:06,387 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:06,387 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:C 2024-11-11T13:36:06,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:06,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:06,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332226489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332226489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332226509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332226509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:06,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:06,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332226649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/5336facef3a647838f8b7357e364ec4a 2024-11-11T13:36:06,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/085275ef10a148e6a04008db093d7aa0 is 50, key is test_row_0/B:col10/1731332166021/Put/seqid=0 2024-11-11T13:36:06,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742065_1241 (size=12001) 2024-11-11T13:36:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T13:36:06,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:06,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:06,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332226799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332226803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332226819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332226819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:06,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:06,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:06,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:07,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/085275ef10a148e6a04008db093d7aa0 2024-11-11T13:36:07,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/3812e5a8d91949c19e55fc346c43496a is 50, key is test_row_0/C:col10/1731332166021/Put/seqid=0 2024-11-11T13:36:07,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:07,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:07,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:07,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742066_1242 (size=12001) 2024-11-11T13:36:07,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332227159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:07,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:07,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:07,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332227312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:07,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332227311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:07,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332227330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:07,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:07,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332227331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:07,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:07,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:07,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:07,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:07,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/3812e5a8d91949c19e55fc346c43496a 2024-11-11T13:36:07,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/5336facef3a647838f8b7357e364ec4a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/5336facef3a647838f8b7357e364ec4a 2024-11-11T13:36:07,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/5336facef3a647838f8b7357e364ec4a, entries=200, sequenceid=80, filesize=14.0 K 2024-11-11T13:36:07,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/085275ef10a148e6a04008db093d7aa0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/085275ef10a148e6a04008db093d7aa0 2024-11-11T13:36:07,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/085275ef10a148e6a04008db093d7aa0, entries=150, sequenceid=80, filesize=11.7 K 2024-11-11T13:36:07,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/3812e5a8d91949c19e55fc346c43496a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/3812e5a8d91949c19e55fc346c43496a 2024-11-11T13:36:07,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/3812e5a8d91949c19e55fc346c43496a, entries=150, sequenceid=80, filesize=11.7 K 2024-11-11T13:36:07,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 7cb6411e18fb1479182e4865079a3fa6 in 1423ms, sequenceid=80, compaction requested=false 2024-11-11T13:36:07,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T13:36:07,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:07,577 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T13:36:07,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/c3e42f96287c4dc0b2ef1f9981902904 is 50, key is test_row_1/A:col10/1731332166162/Put/seqid=0 2024-11-11T13:36:07,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742067_1243 (size=9657) 2024-11-11T13:36:07,601 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/c3e42f96287c4dc0b2ef1f9981902904 2024-11-11T13:36:07,616 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T13:36:07,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/52fc7fab10954878bd8a541a1bcc1bc5 is 50, key is test_row_1/B:col10/1731332166162/Put/seqid=0 2024-11-11T13:36:07,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742068_1244 (size=9657) 2024-11-11T13:36:08,076 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/52fc7fab10954878bd8a541a1bcc1bc5 2024-11-11T13:36:08,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/2e6fc01a34d24fce837015e9d515b6bb is 50, key is test_row_1/C:col10/1731332166162/Put/seqid=0 2024-11-11T13:36:08,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742069_1245 (size=9657) 2024-11-11T13:36:08,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:08,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:08,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332228311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:08,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:08,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332228329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:08,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332228329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:08,332 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in 7cb6411e18fb1479182e4865079a3fa6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:08,332 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in 7cb6411e18fb1479182e4865079a3fa6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:08,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:08,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332228339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332228339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:08,341 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in 7cb6411e18fb1479182e4865079a3fa6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:08,342 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in 7cb6411e18fb1479182e4865079a3fa6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:08,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332228419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:08,547 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/2e6fc01a34d24fce837015e9d515b6bb 2024-11-11T13:36:08,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/c3e42f96287c4dc0b2ef1f9981902904 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e42f96287c4dc0b2ef1f9981902904 2024-11-11T13:36:08,578 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e42f96287c4dc0b2ef1f9981902904, entries=100, sequenceid=91, filesize=9.4 K 2024-11-11T13:36:08,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/52fc7fab10954878bd8a541a1bcc1bc5 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/52fc7fab10954878bd8a541a1bcc1bc5 2024-11-11T13:36:08,589 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/52fc7fab10954878bd8a541a1bcc1bc5, entries=100, sequenceid=91, filesize=9.4 K 2024-11-11T13:36:08,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/2e6fc01a34d24fce837015e9d515b6bb as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/2e6fc01a34d24fce837015e9d515b6bb 2024-11-11T13:36:08,598 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/2e6fc01a34d24fce837015e9d515b6bb, entries=100, sequenceid=91, filesize=9.4 K 2024-11-11T13:36:08,599 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 7cb6411e18fb1479182e4865079a3fa6 in 1022ms, sequenceid=91, compaction requested=true 2024-11-11T13:36:08,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:08,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:08,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-11T13:36:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-11T13:36:08,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T13:36:08,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9950 sec 2024-11-11T13:36:08,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 4.0030 sec 2024-11-11T13:36:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:08,632 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:08,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-11T13:36:08,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/c3e40b63a8ca46b7b602acfffa14ec1f is 50, key is test_row_0/A:col10/1731332168629/Put/seqid=0 2024-11-11T13:36:08,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332228649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:08,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742070_1246 (size=12001) 2024-11-11T13:36:08,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332228759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T13:36:08,780 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:08,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:08,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-11T13:36:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T13:36:08,784 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:08,785 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:08,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T13:36:08,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T13:36:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:08,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:08,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332228969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:09,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/c3e40b63a8ca46b7b602acfffa14ec1f 2024-11-11T13:36:09,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/cbbd813b8113475889f141a9d1e85cb2 is 50, key is test_row_0/B:col10/1731332168629/Put/seqid=0 2024-11-11T13:36:09,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742071_1247 (size=12001) 2024-11-11T13:36:09,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T13:36:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:09,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:09,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:09,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/cbbd813b8113475889f141a9d1e85cb2 2024-11-11T13:36:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T13:36:09,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/9e7d16ae403c49e98b74ab0ae0c0139b is 50, key is test_row_0/C:col10/1731332168629/Put/seqid=0 2024-11-11T13:36:09,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742072_1248 (size=12001) 2024-11-11T13:36:09,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/9e7d16ae403c49e98b74ab0ae0c0139b 2024-11-11T13:36:09,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/c3e40b63a8ca46b7b602acfffa14ec1f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e40b63a8ca46b7b602acfffa14ec1f 2024-11-11T13:36:09,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e40b63a8ca46b7b602acfffa14ec1f, entries=150, sequenceid=120, filesize=11.7 K 2024-11-11T13:36:09,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/cbbd813b8113475889f141a9d1e85cb2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/cbbd813b8113475889f141a9d1e85cb2 2024-11-11T13:36:09,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/cbbd813b8113475889f141a9d1e85cb2, entries=150, sequenceid=120, filesize=11.7 K 2024-11-11T13:36:09,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/9e7d16ae403c49e98b74ab0ae0c0139b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9e7d16ae403c49e98b74ab0ae0c0139b 2024-11-11T13:36:09,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9e7d16ae403c49e98b74ab0ae0c0139b, entries=150, sequenceid=120, filesize=11.7 K 2024-11-11T13:36:09,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 7cb6411e18fb1479182e4865079a3fa6 in 526ms, sequenceid=120, compaction requested=true 2024-11-11T13:36:09,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:09,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:09,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:09,158 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:09,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:09,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:09,158 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:09,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:09,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:09,160 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48103 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:09,160 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:09,160 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/A is initiating minor compaction (all files) 2024-11-11T13:36:09,160 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/B is initiating minor compaction (all files) 2024-11-11T13:36:09,161 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/A in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:09,161 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/B in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:09,161 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/a48044f21e35401fb1752ad39fe98660, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/5336facef3a647838f8b7357e364ec4a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e42f96287c4dc0b2ef1f9981902904, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e40b63a8ca46b7b602acfffa14ec1f] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=47.0 K 2024-11-11T13:36:09,161 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f9f4a86ee2b34d27801735dfa1ae8a0d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/085275ef10a148e6a04008db093d7aa0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/52fc7fab10954878bd8a541a1bcc1bc5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/cbbd813b8113475889f141a9d1e85cb2] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=44.7 K 2024-11-11T13:36:09,161 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f9f4a86ee2b34d27801735dfa1ae8a0d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332165361 2024-11-11T13:36:09,161 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a48044f21e35401fb1752ad39fe98660, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332165361 2024-11-11T13:36:09,161 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 085275ef10a148e6a04008db093d7aa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731332166021 2024-11-11T13:36:09,163 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5336facef3a647838f8b7357e364ec4a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731332166021 2024-11-11T13:36:09,163 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 52fc7fab10954878bd8a541a1bcc1bc5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731332166162 2024-11-11T13:36:09,163 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3e42f96287c4dc0b2ef1f9981902904, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731332166162 2024-11-11T13:36:09,163 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting cbbd813b8113475889f141a9d1e85cb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332168297 2024-11-11T13:36:09,164 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3e40b63a8ca46b7b602acfffa14ec1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332168297 2024-11-11T13:36:09,178 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:09,179 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/df770b3eb35943f89a4b9c46811f452d is 50, key is test_row_0/B:col10/1731332168629/Put/seqid=0 2024-11-11T13:36:09,180 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#A#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:09,180 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/e9730c1177be453f9251ad70ff59fde8 is 50, key is test_row_0/A:col10/1731332168629/Put/seqid=0 2024-11-11T13:36:09,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742073_1249 (size=12241) 2024-11-11T13:36:09,195 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/df770b3eb35943f89a4b9c46811f452d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/df770b3eb35943f89a4b9c46811f452d 2024-11-11T13:36:09,202 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/B of 7cb6411e18fb1479182e4865079a3fa6 into df770b3eb35943f89a4b9c46811f452d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:09,202 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:09,202 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/B, priority=12, startTime=1731332169158; duration=0sec 2024-11-11T13:36:09,202 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:09,202 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:B 2024-11-11T13:36:09,202 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:09,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742074_1250 (size=12241) 2024-11-11T13:36:09,205 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:09,205 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/C is initiating minor compaction (all files) 2024-11-11T13:36:09,205 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/C in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:09,206 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d0cd7bac37c64437aadbce78c458ee64, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/3812e5a8d91949c19e55fc346c43496a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/2e6fc01a34d24fce837015e9d515b6bb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9e7d16ae403c49e98b74ab0ae0c0139b] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=44.7 K 2024-11-11T13:36:09,206 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting d0cd7bac37c64437aadbce78c458ee64, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332165361 2024-11-11T13:36:09,207 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 3812e5a8d91949c19e55fc346c43496a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731332166021 2024-11-11T13:36:09,208 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e6fc01a34d24fce837015e9d515b6bb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731332166162 2024-11-11T13:36:09,208 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e7d16ae403c49e98b74ab0ae0c0139b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332168297 2024-11-11T13:36:09,219 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:09,220 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/13743ca702e34b7cb3be06adfd452a7b is 50, key is test_row_0/C:col10/1731332168629/Put/seqid=0 2024-11-11T13:36:09,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742075_1251 (size=12241) 2024-11-11T13:36:09,238 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/13743ca702e34b7cb3be06adfd452a7b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/13743ca702e34b7cb3be06adfd452a7b 2024-11-11T13:36:09,246 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/C of 7cb6411e18fb1479182e4865079a3fa6 into 13743ca702e34b7cb3be06adfd452a7b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:09,246 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:09,246 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/C, priority=12, startTime=1731332169158; duration=0sec 2024-11-11T13:36:09,246 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:09,246 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:C 2024-11-11T13:36:09,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T13:36:09,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:09,252 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-11T13:36:09,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/bd4b85b618b24811b6107b8bf499d449 is 50, key is test_row_0/A:col10/1731332168647/Put/seqid=0 2024-11-11T13:36:09,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742076_1252 (size=9657) 2024-11-11T13:36:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:09,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:09,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332229375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T13:36:09,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332229489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:09,628 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/e9730c1177be453f9251ad70ff59fde8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/e9730c1177be453f9251ad70ff59fde8 2024-11-11T13:36:09,637 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/A of 7cb6411e18fb1479182e4865079a3fa6 into e9730c1177be453f9251ad70ff59fde8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:09,638 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:09,638 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/A, priority=12, startTime=1731332169158; duration=0sec 2024-11-11T13:36:09,638 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:09,638 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:A 2024-11-11T13:36:09,679 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/bd4b85b618b24811b6107b8bf499d449 2024-11-11T13:36:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/739d8071fe134059b81a057f53487f74 is 50, key is test_row_0/B:col10/1731332168647/Put/seqid=0 2024-11-11T13:36:09,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332229704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:09,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:09,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742077_1253 (size=9657) 2024-11-11T13:36:09,706 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/739d8071fe134059b81a057f53487f74 2024-11-11T13:36:09,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/9723c5bade5948c79e8616dc48f1c218 is 50, key is test_row_0/C:col10/1731332168647/Put/seqid=0 2024-11-11T13:36:09,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742078_1254 (size=9657) 2024-11-11T13:36:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T13:36:10,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332230009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:10,171 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/9723c5bade5948c79e8616dc48f1c218 2024-11-11T13:36:10,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/bd4b85b618b24811b6107b8bf499d449 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/bd4b85b618b24811b6107b8bf499d449 2024-11-11T13:36:10,236 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/bd4b85b618b24811b6107b8bf499d449, entries=100, sequenceid=129, filesize=9.4 K 2024-11-11T13:36:10,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/739d8071fe134059b81a057f53487f74 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/739d8071fe134059b81a057f53487f74 2024-11-11T13:36:10,244 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/739d8071fe134059b81a057f53487f74, entries=100, sequenceid=129, filesize=9.4 K 2024-11-11T13:36:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/9723c5bade5948c79e8616dc48f1c218 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9723c5bade5948c79e8616dc48f1c218 2024-11-11T13:36:10,267 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9723c5bade5948c79e8616dc48f1c218, entries=100, sequenceid=129, filesize=9.4 K 2024-11-11T13:36:10,269 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 7cb6411e18fb1479182e4865079a3fa6 in 1017ms, sequenceid=129, compaction requested=false 2024-11-11T13:36:10,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-11T13:36:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-11T13:36:10,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-11T13:36:10,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4890 sec 2024-11-11T13:36:10,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.5030 sec 2024-11-11T13:36:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:10,342 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:10,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-11T13:36:10,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/2f4afb64d1b4459f8618ec75314260a1 is 50, key is test_row_0/A:col10/1731332169365/Put/seqid=0 2024-11-11T13:36:10,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332230349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:10,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332230356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:10,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332230360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,367 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in 7cb6411e18fb1479182e4865079a3fa6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:10,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742079_1255 (size=14541) 2024-11-11T13:36:10,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332230369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,371 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in 7cb6411e18fb1479182e4865079a3fa6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:10,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332230460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:10,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332230470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:10,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332230520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:10,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332230669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:10,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332230679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:10,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/2f4afb64d1b4459f8618ec75314260a1 2024-11-11T13:36:10,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/c3587d313f614e80b6b8cf16236418f1 is 50, key is test_row_0/B:col10/1731332169365/Put/seqid=0 2024-11-11T13:36:10,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742080_1256 (size=12151) 2024-11-11T13:36:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T13:36:10,929 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:10,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-11T13:36:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T13:36:10,932 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:10,932 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:10,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:10,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332230979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:10,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332230989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T13:36:11,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T13:36:11,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:11,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/c3587d313f614e80b6b8cf16236418f1 2024-11-11T13:36:11,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/f61df8584f0f4b95822325feaa29dd43 is 50, key is test_row_0/C:col10/1731332169365/Put/seqid=0 2024-11-11T13:36:11,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742081_1257 (size=12151) 2024-11-11T13:36:11,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T13:36:11,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:11,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T13:36:11,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T13:36:11,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:11,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332231490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:11,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332231500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:11,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:11,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332231535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:11,543 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in 7cb6411e18fb1479182e4865079a3fa6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:11,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:11,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:11,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:11,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T13:36:11,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:11,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T13:36:11,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/f61df8584f0f4b95822325feaa29dd43 2024-11-11T13:36:11,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/2f4afb64d1b4459f8618ec75314260a1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/2f4afb64d1b4459f8618ec75314260a1 2024-11-11T13:36:11,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/2f4afb64d1b4459f8618ec75314260a1, entries=200, sequenceid=160, filesize=14.2 K 2024-11-11T13:36:11,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/c3587d313f614e80b6b8cf16236418f1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c3587d313f614e80b6b8cf16236418f1 2024-11-11T13:36:11,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c3587d313f614e80b6b8cf16236418f1, entries=150, sequenceid=160, filesize=11.9 K 2024-11-11T13:36:11,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/f61df8584f0f4b95822325feaa29dd43 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f61df8584f0f4b95822325feaa29dd43 2024-11-11T13:36:11,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f61df8584f0f4b95822325feaa29dd43, entries=150, sequenceid=160, filesize=11.9 K 2024-11-11T13:36:11,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 7cb6411e18fb1479182e4865079a3fa6 in 1319ms, sequenceid=160, compaction requested=true 2024-11-11T13:36:11,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:11,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:11,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:11,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:11,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:11,662 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:11,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:11,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:11,662 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:11,663 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:11,663 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:11,663 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/B is initiating minor compaction (all files) 2024-11-11T13:36:11,663 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/A is initiating minor compaction (all files) 2024-11-11T13:36:11,664 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/A in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,664 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/B in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,664 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/e9730c1177be453f9251ad70ff59fde8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/bd4b85b618b24811b6107b8bf499d449, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/2f4afb64d1b4459f8618ec75314260a1] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=35.6 K 2024-11-11T13:36:11,664 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/df770b3eb35943f89a4b9c46811f452d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/739d8071fe134059b81a057f53487f74, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c3587d313f614e80b6b8cf16236418f1] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=33.3 K 2024-11-11T13:36:11,664 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9730c1177be453f9251ad70ff59fde8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332168297 2024-11-11T13:36:11,664 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting df770b3eb35943f89a4b9c46811f452d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332168297 2024-11-11T13:36:11,664 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd4b85b618b24811b6107b8bf499d449, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731332168642 2024-11-11T13:36:11,664 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 739d8071fe134059b81a057f53487f74, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731332168642 2024-11-11T13:36:11,664 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f4afb64d1b4459f8618ec75314260a1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731332169352 2024-11-11T13:36:11,665 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c3587d313f614e80b6b8cf16236418f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731332169352 2024-11-11T13:36:11,673 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#A#compaction#216 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:11,673 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/b7204c293efe4603a1246c97ab727934 is 50, key is test_row_0/A:col10/1731332169365/Put/seqid=0 2024-11-11T13:36:11,675 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:11,676 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/7aaec148c9234d1e99a757c4ed0151cc is 50, key is test_row_0/B:col10/1731332169365/Put/seqid=0 2024-11-11T13:36:11,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742083_1259 (size=12493) 2024-11-11T13:36:11,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742082_1258 (size=12493) 2024-11-11T13:36:11,712 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/b7204c293efe4603a1246c97ab727934 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b7204c293efe4603a1246c97ab727934 2024-11-11T13:36:11,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T13:36:11,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,714 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-11T13:36:11,719 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/A of 7cb6411e18fb1479182e4865079a3fa6 into b7204c293efe4603a1246c97ab727934(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:11,719 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:11,719 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/A, priority=13, startTime=1731332171662; duration=0sec 2024-11-11T13:36:11,719 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:11,719 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:A 2024-11-11T13:36:11,719 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/085fa76e781f4a3cbdee12d3050e665b is 50, key is test_row_0/A:col10/1731332170347/Put/seqid=0 2024-11-11T13:36:11,721 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:11,721 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/C is initiating minor compaction (all files) 2024-11-11T13:36:11,721 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/C in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:11,721 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/13743ca702e34b7cb3be06adfd452a7b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9723c5bade5948c79e8616dc48f1c218, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f61df8584f0f4b95822325feaa29dd43] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=33.3 K 2024-11-11T13:36:11,722 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13743ca702e34b7cb3be06adfd452a7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731332168297 2024-11-11T13:36:11,722 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9723c5bade5948c79e8616dc48f1c218, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731332168642 2024-11-11T13:36:11,722 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f61df8584f0f4b95822325feaa29dd43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731332169352 2024-11-11T13:36:11,731 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:11,732 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/012c448e32ee4f5a98dca834bbaef3a1 is 50, key is test_row_0/C:col10/1731332169365/Put/seqid=0 2024-11-11T13:36:11,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742084_1260 (size=12151) 2024-11-11T13:36:11,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742085_1261 (size=12493) 2024-11-11T13:36:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T13:36:12,096 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/7aaec148c9234d1e99a757c4ed0151cc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/7aaec148c9234d1e99a757c4ed0151cc 2024-11-11T13:36:12,101 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/B of 7cb6411e18fb1479182e4865079a3fa6 into 7aaec148c9234d1e99a757c4ed0151cc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:12,101 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:12,101 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/B, priority=13, startTime=1731332171662; duration=0sec 2024-11-11T13:36:12,101 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:12,101 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:B 2024-11-11T13:36:12,138 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/085fa76e781f4a3cbdee12d3050e665b 2024-11-11T13:36:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/a0492661f00d4620ac0ed9d470e9e210 is 50, key is test_row_0/B:col10/1731332170347/Put/seqid=0 2024-11-11T13:36:12,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742086_1262 (size=12151) 2024-11-11T13:36:12,165 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/012c448e32ee4f5a98dca834bbaef3a1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/012c448e32ee4f5a98dca834bbaef3a1 2024-11-11T13:36:12,168 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/a0492661f00d4620ac0ed9d470e9e210 2024-11-11T13:36:12,171 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/C of 7cb6411e18fb1479182e4865079a3fa6 into 012c448e32ee4f5a98dca834bbaef3a1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:12,171 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:12,171 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/C, priority=13, startTime=1731332171662; duration=0sec 2024-11-11T13:36:12,171 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:12,171 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:C 2024-11-11T13:36:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/23a8c5a520ef4527b56dc448cc32236a is 50, key is test_row_0/C:col10/1731332170347/Put/seqid=0 2024-11-11T13:36:12,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742087_1263 (size=12151) 2024-11-11T13:36:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:12,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:12,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332232566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:12,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:12,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332232570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:12,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:12,608 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/23a8c5a520ef4527b56dc448cc32236a 2024-11-11T13:36:12,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/085fa76e781f4a3cbdee12d3050e665b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/085fa76e781f4a3cbdee12d3050e665b 2024-11-11T13:36:12,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332232679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:12,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332232687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:12,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:12,706 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/085fa76e781f4a3cbdee12d3050e665b, entries=150, sequenceid=166, filesize=11.9 K 2024-11-11T13:36:12,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/a0492661f00d4620ac0ed9d470e9e210 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/a0492661f00d4620ac0ed9d470e9e210 2024-11-11T13:36:12,733 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/a0492661f00d4620ac0ed9d470e9e210, entries=150, sequenceid=166, filesize=11.9 K 2024-11-11T13:36:12,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/23a8c5a520ef4527b56dc448cc32236a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/23a8c5a520ef4527b56dc448cc32236a 2024-11-11T13:36:12,740 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/23a8c5a520ef4527b56dc448cc32236a, entries=150, sequenceid=166, filesize=11.9 K 2024-11-11T13:36:12,741 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 7cb6411e18fb1479182e4865079a3fa6 in 1027ms, sequenceid=166, compaction requested=false 2024-11-11T13:36:12,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:12,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:12,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-11T13:36:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-11T13:36:12,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T13:36:12,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8110 sec 2024-11-11T13:36:12,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.8160 sec 2024-11-11T13:36:12,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:12,891 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:12,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-11T13:36:12,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/97a088fd249b43ce883de8e02c33528a is 50, key is test_row_0/A:col10/1731332172890/Put/seqid=0 2024-11-11T13:36:12,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742088_1264 (size=12151) 2024-11-11T13:36:12,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332232897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:12,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332232899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:13,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332233009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T13:36:13,089 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:13,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-11T13:36:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T13:36:13,091 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:13,092 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:13,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T13:36:13,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:13,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332233209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:13,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332233220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:13,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:13,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T13:36:13,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:13,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/97a088fd249b43ce883de8e02c33528a 2024-11-11T13:36:13,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/60c2fa24a6e6454dbad55b01cf74163e is 50, key is test_row_0/B:col10/1731332172890/Put/seqid=0 2024-11-11T13:36:13,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742089_1265 (size=12151) 2024-11-11T13:36:13,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/60c2fa24a6e6454dbad55b01cf74163e 2024-11-11T13:36:13,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/d42011ad50ff4f858623b54a3ff40351 is 50, key is test_row_0/C:col10/1731332172890/Put/seqid=0 2024-11-11T13:36:13,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742090_1266 (size=12151) 2024-11-11T13:36:13,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/d42011ad50ff4f858623b54a3ff40351 2024-11-11T13:36:13,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T13:36:13,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:13,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/97a088fd249b43ce883de8e02c33528a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/97a088fd249b43ce883de8e02c33528a 2024-11-11T13:36:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T13:36:13,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/97a088fd249b43ce883de8e02c33528a, entries=150, sequenceid=200, filesize=11.9 K 2024-11-11T13:36:13,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/60c2fa24a6e6454dbad55b01cf74163e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/60c2fa24a6e6454dbad55b01cf74163e 2024-11-11T13:36:13,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/60c2fa24a6e6454dbad55b01cf74163e, entries=150, sequenceid=200, filesize=11.9 K 2024-11-11T13:36:13,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/d42011ad50ff4f858623b54a3ff40351 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d42011ad50ff4f858623b54a3ff40351 2024-11-11T13:36:13,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d42011ad50ff4f858623b54a3ff40351, entries=150, sequenceid=200, filesize=11.9 K 2024-11-11T13:36:13,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 7cb6411e18fb1479182e4865079a3fa6 in 541ms, sequenceid=200, compaction requested=true 2024-11-11T13:36:13,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:13,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:13,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:13,432 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:13,432 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:13,434 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:13,434 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/A is initiating minor compaction (all files) 2024-11-11T13:36:13,434 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/A in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,434 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b7204c293efe4603a1246c97ab727934, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/085fa76e781f4a3cbdee12d3050e665b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/97a088fd249b43ce883de8e02c33528a] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=35.9 K 2024-11-11T13:36:13,434 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:13,434 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/B is initiating minor compaction (all files) 2024-11-11T13:36:13,434 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/B in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,434 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/7aaec148c9234d1e99a757c4ed0151cc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/a0492661f00d4620ac0ed9d470e9e210, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/60c2fa24a6e6454dbad55b01cf74163e] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=35.9 K 2024-11-11T13:36:13,435 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7204c293efe4603a1246c97ab727934, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731332169352 2024-11-11T13:36:13,435 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aaec148c9234d1e99a757c4ed0151cc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731332169352 2024-11-11T13:36:13,435 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 085fa76e781f4a3cbdee12d3050e665b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731332170346 2024-11-11T13:36:13,435 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting a0492661f00d4620ac0ed9d470e9e210, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731332170346 2024-11-11T13:36:13,436 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97a088fd249b43ce883de8e02c33528a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731332172551 2024-11-11T13:36:13,436 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 60c2fa24a6e6454dbad55b01cf74163e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731332172551 2024-11-11T13:36:13,445 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:13,446 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/6d88699357eb4e9093fe4862725c9984 is 50, key is test_row_0/A:col10/1731332172890/Put/seqid=0 2024-11-11T13:36:13,450 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:13,450 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/ac923184b7cf46ada0afbc4927b96c2b is 50, key is test_row_0/B:col10/1731332172890/Put/seqid=0 2024-11-11T13:36:13,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742091_1267 (size=12595) 2024-11-11T13:36:13,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742092_1268 (size=12595) 2024-11-11T13:36:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:13,547 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:13,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:36:13,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/1c423620a05e4820a96677d144a6f1d0 is 50, key is test_row_0/A:col10/1731332173537/Put/seqid=0 2024-11-11T13:36:13,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742093_1269 (size=14541) 2024-11-11T13:36:13,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T13:36:13,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:13,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332233604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:13,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332233609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:13,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T13:36:13,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:13,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T13:36:13,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332233719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:13,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332233719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:13,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332233723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:13,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:13,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T13:36:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:13,892 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/6d88699357eb4e9093fe4862725c9984 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/6d88699357eb4e9093fe4862725c9984 2024-11-11T13:36:13,895 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/ac923184b7cf46ada0afbc4927b96c2b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/ac923184b7cf46ada0afbc4927b96c2b 2024-11-11T13:36:13,902 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/A of 7cb6411e18fb1479182e4865079a3fa6 into 6d88699357eb4e9093fe4862725c9984(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:13,902 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:13,902 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/A, priority=13, startTime=1731332173432; duration=0sec 2024-11-11T13:36:13,902 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:13,902 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:A 2024-11-11T13:36:13,902 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:13,903 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/B of 7cb6411e18fb1479182e4865079a3fa6 into ac923184b7cf46ada0afbc4927b96c2b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:13,903 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:13,903 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/B, priority=13, startTime=1731332173432; duration=0sec 2024-11-11T13:36:13,904 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:13,904 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:B 2024-11-11T13:36:13,904 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:13,904 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/C is initiating minor compaction (all files) 2024-11-11T13:36:13,904 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/C in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:13,904 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/012c448e32ee4f5a98dca834bbaef3a1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/23a8c5a520ef4527b56dc448cc32236a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d42011ad50ff4f858623b54a3ff40351] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=35.9 K 2024-11-11T13:36:13,905 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 012c448e32ee4f5a98dca834bbaef3a1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731332169352 2024-11-11T13:36:13,905 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23a8c5a520ef4527b56dc448cc32236a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731332170346 2024-11-11T13:36:13,906 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d42011ad50ff4f858623b54a3ff40351, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731332172551 2024-11-11T13:36:13,921 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#C#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:13,922 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/ed51c1dd1dac4cbdb009248f7d12c532 is 50, key is test_row_0/C:col10/1731332172890/Put/seqid=0 2024-11-11T13:36:13,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332233929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:13,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:13,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332233929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:13,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742094_1270 (size=12595) 2024-11-11T13:36:13,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/1c423620a05e4820a96677d144a6f1d0 2024-11-11T13:36:13,957 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/ed51c1dd1dac4cbdb009248f7d12c532 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/ed51c1dd1dac4cbdb009248f7d12c532 2024-11-11T13:36:13,964 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/C of 7cb6411e18fb1479182e4865079a3fa6 into ed51c1dd1dac4cbdb009248f7d12c532(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:13,964 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:13,965 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/C, priority=13, startTime=1731332173433; duration=0sec 2024-11-11T13:36:13,965 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:13,965 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:C 2024-11-11T13:36:13,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/541e631eb22244f4b262a6e7c6177e9a is 50, key is test_row_0/B:col10/1731332173537/Put/seqid=0 2024-11-11T13:36:13,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742095_1271 (size=12151) 2024-11-11T13:36:14,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T13:36:14,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:14,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:14,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T13:36:14,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:14,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:14,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:14,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:14,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T13:36:14,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332234239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:14,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332234239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:14,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T13:36:14,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:14,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:14,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:14,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:14,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:14,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/541e631eb22244f4b262a6e7c6177e9a 2024-11-11T13:36:14,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1a885556a5d043068155f12378729d9a is 50, key is test_row_0/C:col10/1731332173537/Put/seqid=0 2024-11-11T13:36:14,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332234395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:14,399 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in 7cb6411e18fb1479182e4865079a3fa6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:14,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742096_1272 (size=12151) 2024-11-11T13:36:14,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1a885556a5d043068155f12378729d9a 2024-11-11T13:36:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/1c423620a05e4820a96677d144a6f1d0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c423620a05e4820a96677d144a6f1d0 2024-11-11T13:36:14,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332234421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:14,423 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in 7cb6411e18fb1479182e4865079a3fa6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:14,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c423620a05e4820a96677d144a6f1d0, entries=200, sequenceid=211, filesize=14.2 K 2024-11-11T13:36:14,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/541e631eb22244f4b262a6e7c6177e9a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/541e631eb22244f4b262a6e7c6177e9a 2024-11-11T13:36:14,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/541e631eb22244f4b262a6e7c6177e9a, entries=150, sequenceid=211, filesize=11.9 K 2024-11-11T13:36:14,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1a885556a5d043068155f12378729d9a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1a885556a5d043068155f12378729d9a 2024-11-11T13:36:14,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1a885556a5d043068155f12378729d9a, entries=150, sequenceid=211, filesize=11.9 K 2024-11-11T13:36:14,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7cb6411e18fb1479182e4865079a3fa6 in 944ms, sequenceid=211, compaction requested=false 2024-11-11T13:36:14,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:14,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T13:36:14,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:14,503 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:36:14,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/1c9cb5922c4d4de7a9f80da5fa841ef0 is 50, key is test_row_0/A:col10/1731332173591/Put/seqid=0 2024-11-11T13:36:14,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742097_1273 (size=12151) 2024-11-11T13:36:14,547 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/1c9cb5922c4d4de7a9f80da5fa841ef0 2024-11-11T13:36:14,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/894b53bec6ed4ef68112dc2e70e84273 is 50, key is test_row_0/B:col10/1731332173591/Put/seqid=0 2024-11-11T13:36:14,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742098_1274 (size=12151) 2024-11-11T13:36:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:14,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332234752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:14,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332234754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:14,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332234754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:14,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332234860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:14,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332234863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:14,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332234871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:14,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:14,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:14,968 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/894b53bec6ed4ef68112dc2e70e84273 2024-11-11T13:36:14,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1ea9fcfb715040dab8fd058948d88ad9 is 50, key is test_row_0/C:col10/1731332173591/Put/seqid=0 2024-11-11T13:36:15,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742099_1275 (size=12151) 2024-11-11T13:36:15,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332235080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:15,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332235080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332235080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T13:36:15,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332235389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:15,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332235389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:15,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332235390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:15,407 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1ea9fcfb715040dab8fd058948d88ad9 2024-11-11T13:36:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/1c9cb5922c4d4de7a9f80da5fa841ef0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c9cb5922c4d4de7a9f80da5fa841ef0 2024-11-11T13:36:15,415 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c9cb5922c4d4de7a9f80da5fa841ef0, entries=150, sequenceid=239, filesize=11.9 K 2024-11-11T13:36:15,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/894b53bec6ed4ef68112dc2e70e84273 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/894b53bec6ed4ef68112dc2e70e84273 2024-11-11T13:36:15,420 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/894b53bec6ed4ef68112dc2e70e84273, entries=150, sequenceid=239, filesize=11.9 K 2024-11-11T13:36:15,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/1ea9fcfb715040dab8fd058948d88ad9 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1ea9fcfb715040dab8fd058948d88ad9 2024-11-11T13:36:15,425 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1ea9fcfb715040dab8fd058948d88ad9, entries=150, sequenceid=239, filesize=11.9 K 2024-11-11T13:36:15,426 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 7cb6411e18fb1479182e4865079a3fa6 in 923ms, sequenceid=239, compaction requested=true 2024-11-11T13:36:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-11T13:36:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-11T13:36:15,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T13:36:15,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3360 sec 2024-11-11T13:36:15,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.3410 sec 2024-11-11T13:36:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:15,903 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:15,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T13:36:15,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/d51d5687b4124d8b827fa02f01effbea is 50, key is test_row_0/A:col10/1731332174752/Put/seqid=0 2024-11-11T13:36:15,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742100_1276 (size=12151) 2024-11-11T13:36:15,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/d51d5687b4124d8b827fa02f01effbea 2024-11-11T13:36:15,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332235950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:15,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332235952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:15,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332235953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:15,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/f672333bd7614ac1bb625d5e7e50de3d is 50, key is test_row_0/B:col10/1731332174752/Put/seqid=0 2024-11-11T13:36:15,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742101_1277 (size=12151) 2024-11-11T13:36:16,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332236059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332236059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:16,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332236069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:16,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332236269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332236269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:16,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332236280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:16,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/f672333bd7614ac1bb625d5e7e50de3d 2024-11-11T13:36:16,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/8eb79cdc2aa34a16a7ec7d407d1d9de9 is 50, key is test_row_0/C:col10/1731332174752/Put/seqid=0 2024-11-11T13:36:16,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742102_1278 (size=12151) 2024-11-11T13:36:16,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332236579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:16,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332236583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:16,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332236589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:16,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/8eb79cdc2aa34a16a7ec7d407d1d9de9 2024-11-11T13:36:16,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/d51d5687b4124d8b827fa02f01effbea as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/d51d5687b4124d8b827fa02f01effbea 2024-11-11T13:36:16,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/d51d5687b4124d8b827fa02f01effbea, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T13:36:16,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/f672333bd7614ac1bb625d5e7e50de3d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f672333bd7614ac1bb625d5e7e50de3d 2024-11-11T13:36:16,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f672333bd7614ac1bb625d5e7e50de3d, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T13:36:16,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/8eb79cdc2aa34a16a7ec7d407d1d9de9 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/8eb79cdc2aa34a16a7ec7d407d1d9de9 2024-11-11T13:36:16,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/8eb79cdc2aa34a16a7ec7d407d1d9de9, entries=150, sequenceid=254, filesize=11.9 K 2024-11-11T13:36:16,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 7cb6411e18fb1479182e4865079a3fa6 in 976ms, sequenceid=254, compaction requested=true 2024-11-11T13:36:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:16,879 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:36:16,881 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:16,881 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/A is initiating minor compaction (all files) 2024-11-11T13:36:16,881 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/A in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:16,881 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/6d88699357eb4e9093fe4862725c9984, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c423620a05e4820a96677d144a6f1d0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c9cb5922c4d4de7a9f80da5fa841ef0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/d51d5687b4124d8b827fa02f01effbea] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=50.2 K 2024-11-11T13:36:16,882 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d88699357eb4e9093fe4862725c9984, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731332172551 2024-11-11T13:36:16,882 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:16,882 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c423620a05e4820a96677d144a6f1d0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731332172895 2024-11-11T13:36:16,882 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c9cb5922c4d4de7a9f80da5fa841ef0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731332173591 2024-11-11T13:36:16,883 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d51d5687b4124d8b827fa02f01effbea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731332174752 2024-11-11T13:36:16,883 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:16,883 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/B is initiating minor compaction (all files) 2024-11-11T13:36:16,884 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/B in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:16,885 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/ac923184b7cf46ada0afbc4927b96c2b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/541e631eb22244f4b262a6e7c6177e9a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/894b53bec6ed4ef68112dc2e70e84273, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f672333bd7614ac1bb625d5e7e50de3d] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=47.9 K 2024-11-11T13:36:16,885 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting ac923184b7cf46ada0afbc4927b96c2b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731332172551 2024-11-11T13:36:16,886 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 541e631eb22244f4b262a6e7c6177e9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731332173537 2024-11-11T13:36:16,887 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 894b53bec6ed4ef68112dc2e70e84273, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731332173591 2024-11-11T13:36:16,887 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f672333bd7614ac1bb625d5e7e50de3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731332174752 2024-11-11T13:36:16,894 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:16,895 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/24b25a59f5464072b176ff3c4250e55b is 50, key is test_row_0/A:col10/1731332174752/Put/seqid=0 2024-11-11T13:36:16,906 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:16,906 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/78bc9f1f7523469f82a184cd16831fea is 50, key is test_row_0/B:col10/1731332174752/Put/seqid=0 2024-11-11T13:36:16,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742103_1279 (size=12731) 2024-11-11T13:36:16,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742104_1280 (size=12731) 2024-11-11T13:36:16,998 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/24b25a59f5464072b176ff3c4250e55b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/24b25a59f5464072b176ff3c4250e55b 2024-11-11T13:36:17,000 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/78bc9f1f7523469f82a184cd16831fea as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/78bc9f1f7523469f82a184cd16831fea 2024-11-11T13:36:17,012 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/B of 7cb6411e18fb1479182e4865079a3fa6 into 78bc9f1f7523469f82a184cd16831fea(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:17,012 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:17,012 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/B, priority=12, startTime=1731332176879; duration=0sec 2024-11-11T13:36:17,013 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:17,013 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:B 2024-11-11T13:36:17,013 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:17,014 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/A of 7cb6411e18fb1479182e4865079a3fa6 into 24b25a59f5464072b176ff3c4250e55b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:17,014 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:17,014 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/A, priority=12, startTime=1731332176879; duration=0sec 2024-11-11T13:36:17,014 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:17,014 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:A 2024-11-11T13:36:17,020 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:17,020 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/C is initiating minor compaction (all files) 2024-11-11T13:36:17,020 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/C in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:17,020 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/ed51c1dd1dac4cbdb009248f7d12c532, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1a885556a5d043068155f12378729d9a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1ea9fcfb715040dab8fd058948d88ad9, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/8eb79cdc2aa34a16a7ec7d407d1d9de9] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=47.9 K 2024-11-11T13:36:17,022 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting ed51c1dd1dac4cbdb009248f7d12c532, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731332172551 2024-11-11T13:36:17,025 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a885556a5d043068155f12378729d9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731332173537 2024-11-11T13:36:17,028 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ea9fcfb715040dab8fd058948d88ad9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731332173591 2024-11-11T13:36:17,029 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eb79cdc2aa34a16a7ec7d407d1d9de9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731332174752 2024-11-11T13:36:17,055 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:17,056 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/cb04f8f6e70d4eb2b1ba942232753623 is 50, key is test_row_0/C:col10/1731332174752/Put/seqid=0 2024-11-11T13:36:17,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742105_1281 (size=12731) 2024-11-11T13:36:17,100 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:17,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T13:36:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:17,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/b0f8af95ea5c49f894706dc95a841fe8 is 50, key is test_row_0/A:col10/1731332175934/Put/seqid=0 2024-11-11T13:36:17,136 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/cb04f8f6e70d4eb2b1ba942232753623 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/cb04f8f6e70d4eb2b1ba942232753623 2024-11-11T13:36:17,144 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/C of 7cb6411e18fb1479182e4865079a3fa6 into cb04f8f6e70d4eb2b1ba942232753623(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:17,144 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:17,144 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/C, priority=12, startTime=1731332176879; duration=0sec 2024-11-11T13:36:17,144 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:17,144 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:C 2024-11-11T13:36:17,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742106_1282 (size=14741) 2024-11-11T13:36:17,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332237188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332237188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332237191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T13:36:17,249 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:17,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-11T13:36:17,251 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T13:36:17,252 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:17,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:17,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332237300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332237303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332237309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T13:36:17,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T13:36:17,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:17,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:17,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:17,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332237509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332237520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332237520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T13:36:17,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:17,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:17,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:17,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/b0f8af95ea5c49f894706dc95a841fe8 2024-11-11T13:36:17,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/8d80c072b8754621afc3273360ccd2fd is 50, key is test_row_0/B:col10/1731332175934/Put/seqid=0 2024-11-11T13:36:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T13:36:17,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742107_1283 (size=12301) 2024-11-11T13:36:17,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T13:36:17,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:17,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:17,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:17,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332237823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332237829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332237829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:17,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T13:36:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:17,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T13:36:17,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/8d80c072b8754621afc3273360ccd2fd 2024-11-11T13:36:17,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/adfed80e95a94fc9afc62d3e58dee8dd is 50, key is test_row_0/C:col10/1731332175934/Put/seqid=0 2024-11-11T13:36:17,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742108_1284 (size=12301) 2024-11-11T13:36:18,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T13:36:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:18,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:18,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T13:36:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:18,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:18,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T13:36:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:18,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:18,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332238339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:18,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332238339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:18,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332238340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T13:36:18,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/adfed80e95a94fc9afc62d3e58dee8dd 2024-11-11T13:36:18,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/b0f8af95ea5c49f894706dc95a841fe8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b0f8af95ea5c49f894706dc95a841fe8 2024-11-11T13:36:18,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b0f8af95ea5c49f894706dc95a841fe8, entries=200, sequenceid=278, filesize=14.4 K 2024-11-11T13:36:18,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/8d80c072b8754621afc3273360ccd2fd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8d80c072b8754621afc3273360ccd2fd 2024-11-11T13:36:18,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8d80c072b8754621afc3273360ccd2fd, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T13:36:18,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/adfed80e95a94fc9afc62d3e58dee8dd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/adfed80e95a94fc9afc62d3e58dee8dd 2024-11-11T13:36:18,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/adfed80e95a94fc9afc62d3e58dee8dd, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T13:36:18,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 7cb6411e18fb1479182e4865079a3fa6 in 1323ms, sequenceid=278, compaction requested=false 2024-11-11T13:36:18,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:18,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T13:36:18,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:18,484 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T13:36:18,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/f85dbb0443864280b56f084f36f6c3f8 is 50, key is test_row_0/A:col10/1731332177149/Put/seqid=0 2024-11-11T13:36:18,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742109_1285 (size=12301) 2024-11-11T13:36:18,916 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/f85dbb0443864280b56f084f36f6c3f8 2024-11-11T13:36:18,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/4e0d0170ed27428db2430a1b2df1a9bd is 50, key is test_row_0/B:col10/1731332177149/Put/seqid=0 2024-11-11T13:36:18,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742110_1286 (size=12301) 2024-11-11T13:36:19,365 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/4e0d0170ed27428db2430a1b2df1a9bd 2024-11-11T13:36:19,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/d5caac23bc0047b3abb845518d05c9e4 is 50, key is test_row_0/C:col10/1731332177149/Put/seqid=0 2024-11-11T13:36:19,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742111_1287 (size=12301) 2024-11-11T13:36:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:19,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T13:36:19,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332239417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:19,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332239423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332239423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:19,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332239529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332239529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:19,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332239539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:19,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332239743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332239743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:19,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332239749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:19,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:19,779 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/d5caac23bc0047b3abb845518d05c9e4 2024-11-11T13:36:19,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/f85dbb0443864280b56f084f36f6c3f8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/f85dbb0443864280b56f084f36f6c3f8 2024-11-11T13:36:19,791 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/f85dbb0443864280b56f084f36f6c3f8, entries=150, sequenceid=293, filesize=12.0 K 2024-11-11T13:36:19,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/4e0d0170ed27428db2430a1b2df1a9bd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/4e0d0170ed27428db2430a1b2df1a9bd 2024-11-11T13:36:19,795 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/4e0d0170ed27428db2430a1b2df1a9bd, entries=150, sequenceid=293, filesize=12.0 K 2024-11-11T13:36:19,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/d5caac23bc0047b3abb845518d05c9e4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d5caac23bc0047b3abb845518d05c9e4 2024-11-11T13:36:19,800 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d5caac23bc0047b3abb845518d05c9e4, entries=150, sequenceid=293, filesize=12.0 K 2024-11-11T13:36:19,801 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 7cb6411e18fb1479182e4865079a3fa6 in 1317ms, sequenceid=293, compaction requested=true 2024-11-11T13:36:19,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:19,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:19,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-11T13:36:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-11T13:36:19,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T13:36:19,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5500 sec 2024-11-11T13:36:19,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.5540 sec 2024-11-11T13:36:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:20,065 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:20,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T13:36:20,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/3a3882dfb7c44a29a1c0b325e2f781df is 50, key is test_row_0/A:col10/1731332179421/Put/seqid=0 2024-11-11T13:36:20,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332240103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332240106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332240107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742112_1288 (size=12301) 2024-11-11T13:36:20,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332240219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332240219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332240220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332240429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332240429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332240430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/3a3882dfb7c44a29a1c0b325e2f781df 2024-11-11T13:36:20,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/8dd53baaf7ee4574af1ee780c7491f38 is 50, key is test_row_0/B:col10/1731332179421/Put/seqid=0 2024-11-11T13:36:20,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742113_1289 (size=12301) 2024-11-11T13:36:20,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332240740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332240740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332240740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:20,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/8dd53baaf7ee4574af1ee780c7491f38 2024-11-11T13:36:20,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/69f6feb999ce4d758a4cd5a617b81a6e is 50, key is test_row_0/C:col10/1731332179421/Put/seqid=0 2024-11-11T13:36:21,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742114_1290 (size=12301) 2024-11-11T13:36:21,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332241249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:21,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332241249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:21,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332241250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T13:36:21,419 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:21,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T13:36:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T13:36:21,423 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:21,424 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:21,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:21,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/69f6feb999ce4d758a4cd5a617b81a6e 2024-11-11T13:36:21,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/3a3882dfb7c44a29a1c0b325e2f781df as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/3a3882dfb7c44a29a1c0b325e2f781df 2024-11-11T13:36:21,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/3a3882dfb7c44a29a1c0b325e2f781df, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T13:36:21,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/8dd53baaf7ee4574af1ee780c7491f38 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8dd53baaf7ee4574af1ee780c7491f38 2024-11-11T13:36:21,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8dd53baaf7ee4574af1ee780c7491f38, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T13:36:21,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/69f6feb999ce4d758a4cd5a617b81a6e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/69f6feb999ce4d758a4cd5a617b81a6e 2024-11-11T13:36:21,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/69f6feb999ce4d758a4cd5a617b81a6e, entries=150, sequenceid=316, filesize=12.0 K 2024-11-11T13:36:21,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 7cb6411e18fb1479182e4865079a3fa6 in 1400ms, sequenceid=316, compaction requested=true 2024-11-11T13:36:21,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:21,465 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:21,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:21,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:21,467 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:21,467 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52074 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:21,467 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/A is initiating minor compaction (all files) 2024-11-11T13:36:21,467 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/A in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:21,468 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/24b25a59f5464072b176ff3c4250e55b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b0f8af95ea5c49f894706dc95a841fe8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/f85dbb0443864280b56f084f36f6c3f8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/3a3882dfb7c44a29a1c0b325e2f781df] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=50.9 K 2024-11-11T13:36:21,468 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24b25a59f5464072b176ff3c4250e55b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731332174752 2024-11-11T13:36:21,468 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:21,468 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/B is initiating minor compaction (all files) 2024-11-11T13:36:21,468 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/B in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:21,469 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/78bc9f1f7523469f82a184cd16831fea, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8d80c072b8754621afc3273360ccd2fd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/4e0d0170ed27428db2430a1b2df1a9bd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8dd53baaf7ee4574af1ee780c7491f38] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=48.5 K 2024-11-11T13:36:21,469 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0f8af95ea5c49f894706dc95a841fe8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731332175934 2024-11-11T13:36:21,469 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f85dbb0443864280b56f084f36f6c3f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731332177135 2024-11-11T13:36:21,469 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 78bc9f1f7523469f82a184cd16831fea, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731332174752 2024-11-11T13:36:21,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:21,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:21,469 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d80c072b8754621afc3273360ccd2fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731332175934 2024-11-11T13:36:21,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:21,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:21,469 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a3882dfb7c44a29a1c0b325e2f781df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332179398 2024-11-11T13:36:21,470 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e0d0170ed27428db2430a1b2df1a9bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731332177135 2024-11-11T13:36:21,470 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dd53baaf7ee4574af1ee780c7491f38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332179398 2024-11-11T13:36:21,479 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:21,480 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/0f8fc739b6b6432b8fb8bf83f8b97046 is 50, key is test_row_0/B:col10/1731332179421/Put/seqid=0 2024-11-11T13:36:21,486 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:21,486 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/bc7cc84eca6a41f5ac5d0edddead290f is 50, key is test_row_0/A:col10/1731332179421/Put/seqid=0 2024-11-11T13:36:21,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742115_1291 (size=13017) 2024-11-11T13:36:21,508 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/0f8fc739b6b6432b8fb8bf83f8b97046 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/0f8fc739b6b6432b8fb8bf83f8b97046 2024-11-11T13:36:21,525 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/B of 7cb6411e18fb1479182e4865079a3fa6 into 0f8fc739b6b6432b8fb8bf83f8b97046(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:21,525 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:21,525 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/B, priority=12, startTime=1731332181466; duration=0sec 2024-11-11T13:36:21,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742116_1292 (size=13017) 2024-11-11T13:36:21,525 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:21,525 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:B 2024-11-11T13:36:21,525 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T13:36:21,542 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:21,542 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/C is initiating minor compaction (all files) 2024-11-11T13:36:21,544 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/C in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:21,544 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/cb04f8f6e70d4eb2b1ba942232753623, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/adfed80e95a94fc9afc62d3e58dee8dd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d5caac23bc0047b3abb845518d05c9e4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/69f6feb999ce4d758a4cd5a617b81a6e] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=48.5 K 2024-11-11T13:36:21,545 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/bc7cc84eca6a41f5ac5d0edddead290f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/bc7cc84eca6a41f5ac5d0edddead290f 2024-11-11T13:36:21,545 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting cb04f8f6e70d4eb2b1ba942232753623, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731332174752 2024-11-11T13:36:21,547 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting adfed80e95a94fc9afc62d3e58dee8dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731332175934 2024-11-11T13:36:21,554 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/A of 7cb6411e18fb1479182e4865079a3fa6 into bc7cc84eca6a41f5ac5d0edddead290f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:21,554 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting d5caac23bc0047b3abb845518d05c9e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731332177135 2024-11-11T13:36:21,554 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:21,554 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/A, priority=12, startTime=1731332181465; duration=0sec 2024-11-11T13:36:21,554 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:21,554 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:A 2024-11-11T13:36:21,554 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 69f6feb999ce4d758a4cd5a617b81a6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332179398 2024-11-11T13:36:21,572 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:21,572 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/401fd369255240808245becd5d02fe15 is 50, key is test_row_0/C:col10/1731332179421/Put/seqid=0 2024-11-11T13:36:21,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T13:36:21,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:21,583 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T13:36:21,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/fbe44a1905d74a08913874123f52918a is 50, key is test_row_0/A:col10/1731332180099/Put/seqid=0 2024-11-11T13:36:21,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742117_1293 (size=13017) 2024-11-11T13:36:21,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742118_1294 (size=12301) 2024-11-11T13:36:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T13:36:22,021 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/401fd369255240808245becd5d02fe15 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/401fd369255240808245becd5d02fe15 2024-11-11T13:36:22,027 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/C of 7cb6411e18fb1479182e4865079a3fa6 into 401fd369255240808245becd5d02fe15(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:22,027 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:22,027 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/C, priority=12, startTime=1731332181469; duration=0sec 2024-11-11T13:36:22,028 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:22,028 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:C 2024-11-11T13:36:22,037 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/fbe44a1905d74a08913874123f52918a 2024-11-11T13:36:22,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/89b1c6ddcd4a4b4c8a67bf16becfee62 is 50, key is test_row_0/B:col10/1731332180099/Put/seqid=0 2024-11-11T13:36:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T13:36:22,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742119_1295 (size=12301) 2024-11-11T13:36:22,060 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/89b1c6ddcd4a4b4c8a67bf16becfee62 2024-11-11T13:36:22,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/09a55dff35db417e99e0867c7c58cc02 is 50, key is test_row_0/C:col10/1731332180099/Put/seqid=0 2024-11-11T13:36:22,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742120_1296 (size=12301) 2024-11-11T13:36:22,141 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/09a55dff35db417e99e0867c7c58cc02 2024-11-11T13:36:22,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/fbe44a1905d74a08913874123f52918a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/fbe44a1905d74a08913874123f52918a 2024-11-11T13:36:22,176 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/fbe44a1905d74a08913874123f52918a, entries=150, sequenceid=332, filesize=12.0 K 2024-11-11T13:36:22,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/89b1c6ddcd4a4b4c8a67bf16becfee62 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/89b1c6ddcd4a4b4c8a67bf16becfee62 2024-11-11T13:36:22,183 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/89b1c6ddcd4a4b4c8a67bf16becfee62, entries=150, sequenceid=332, filesize=12.0 K 2024-11-11T13:36:22,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/09a55dff35db417e99e0867c7c58cc02 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/09a55dff35db417e99e0867c7c58cc02 2024-11-11T13:36:22,197 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/09a55dff35db417e99e0867c7c58cc02, entries=150, sequenceid=332, filesize=12.0 K 2024-11-11T13:36:22,198 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 7cb6411e18fb1479182e4865079a3fa6 in 615ms, sequenceid=332, compaction requested=false 2024-11-11T13:36:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:22,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T13:36:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T13:36:22,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T13:36:22,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 776 msec 2024-11-11T13:36:22,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 784 msec 2024-11-11T13:36:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:22,274 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:22,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T13:36:22,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/540ce077c8a54a53bdca5cb5673d7f78 is 50, key is test_row_0/A:col10/1731332182262/Put/seqid=0 2024-11-11T13:36:22,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742121_1297 (size=12301) 2024-11-11T13:36:22,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/540ce077c8a54a53bdca5cb5673d7f78 2024-11-11T13:36:22,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/91e0c45e258845c095c54bd81cfeda60 is 50, key is test_row_0/B:col10/1731332182262/Put/seqid=0 2024-11-11T13:36:22,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332242315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:22,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332242318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:22,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332242320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:22,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742122_1298 (size=12301) 2024-11-11T13:36:22,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332242429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332242429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332242430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T13:36:22,559 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:22,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-11T13:36:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T13:36:22,562 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:22,563 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:22,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332242639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332242639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:22,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332242641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T13:36:22,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T13:36:22,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:22,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:22,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:22,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:22,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:22,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/91e0c45e258845c095c54bd81cfeda60 2024-11-11T13:36:22,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/978b1ac2546f46f2b72dabb1edb17c7a is 50, key is test_row_0/C:col10/1731332182262/Put/seqid=0 2024-11-11T13:36:22,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742123_1299 (size=12301) 2024-11-11T13:36:22,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/978b1ac2546f46f2b72dabb1edb17c7a 2024-11-11T13:36:22,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/540ce077c8a54a53bdca5cb5673d7f78 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/540ce077c8a54a53bdca5cb5673d7f78 2024-11-11T13:36:22,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/540ce077c8a54a53bdca5cb5673d7f78, entries=150, sequenceid=346, filesize=12.0 K 2024-11-11T13:36:22,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/91e0c45e258845c095c54bd81cfeda60 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/91e0c45e258845c095c54bd81cfeda60 2024-11-11T13:36:22,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/91e0c45e258845c095c54bd81cfeda60, entries=150, sequenceid=346, filesize=12.0 K 2024-11-11T13:36:22,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/978b1ac2546f46f2b72dabb1edb17c7a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/978b1ac2546f46f2b72dabb1edb17c7a 2024-11-11T13:36:22,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/978b1ac2546f46f2b72dabb1edb17c7a, entries=150, sequenceid=346, filesize=12.0 K 2024-11-11T13:36:22,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 7cb6411e18fb1479182e4865079a3fa6 in 549ms, sequenceid=346, compaction requested=true 2024-11-11T13:36:22,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:22,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:22,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:22,824 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:22,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:22,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:22,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:22,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:36:22,824 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:22,825 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:22,825 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:22,825 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/A is initiating minor compaction (all files) 2024-11-11T13:36:22,825 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/B is initiating minor compaction (all files) 2024-11-11T13:36:22,825 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/B in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:22,825 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/A in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:22,825 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/0f8fc739b6b6432b8fb8bf83f8b97046, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/89b1c6ddcd4a4b4c8a67bf16becfee62, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/91e0c45e258845c095c54bd81cfeda60] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=36.7 K 2024-11-11T13:36:22,825 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/bc7cc84eca6a41f5ac5d0edddead290f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/fbe44a1905d74a08913874123f52918a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/540ce077c8a54a53bdca5cb5673d7f78] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=36.7 K 2024-11-11T13:36:22,825 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc7cc84eca6a41f5ac5d0edddead290f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332179398 2024-11-11T13:36:22,825 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8fc739b6b6432b8fb8bf83f8b97046, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332179398 2024-11-11T13:36:22,826 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe44a1905d74a08913874123f52918a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731332180099 2024-11-11T13:36:22,826 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 89b1c6ddcd4a4b4c8a67bf16becfee62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731332180099 2024-11-11T13:36:22,826 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 540ce077c8a54a53bdca5cb5673d7f78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731332182262 2024-11-11T13:36:22,826 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 91e0c45e258845c095c54bd81cfeda60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731332182262 2024-11-11T13:36:22,842 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#B#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:22,842 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/87e702c8ece0477b97a95e586c938803 is 50, key is test_row_0/B:col10/1731332182262/Put/seqid=0 2024-11-11T13:36:22,845 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:22,845 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/1bbb7a74c93349e0a10026920ca5ddbe is 50, key is test_row_0/A:col10/1731332182262/Put/seqid=0 2024-11-11T13:36:22,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742124_1300 (size=13119) 2024-11-11T13:36:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T13:36:22,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T13:36:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:22,886 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T13:36:22,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742125_1301 (size=13119) 2024-11-11T13:36:22,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/6d226b6c6a0e4dc0a6bb4d363ef3fd5d is 50, key is test_row_0/A:col10/1731332182316/Put/seqid=0 2024-11-11T13:36:22,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742126_1302 (size=12301) 2024-11-11T13:36:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:22,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. as already flushing 2024-11-11T13:36:22,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332242984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:22,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332242988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:22,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332242989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:23,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332243100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:23,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332243100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:23,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332243100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T13:36:23,288 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/87e702c8ece0477b97a95e586c938803 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/87e702c8ece0477b97a95e586c938803 2024-11-11T13:36:23,299 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/B of 7cb6411e18fb1479182e4865079a3fa6 into 87e702c8ece0477b97a95e586c938803(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:23,299 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:23,299 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/B, priority=13, startTime=1731332182824; duration=0sec 2024-11-11T13:36:23,299 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:23,299 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:B 2024-11-11T13:36:23,299 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:23,301 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:23,301 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/C is initiating minor compaction (all files) 2024-11-11T13:36:23,302 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/C in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:23,302 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/401fd369255240808245becd5d02fe15, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/09a55dff35db417e99e0867c7c58cc02, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/978b1ac2546f46f2b72dabb1edb17c7a] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=36.7 K 2024-11-11T13:36:23,302 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 401fd369255240808245becd5d02fe15, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731332179398 2024-11-11T13:36:23,302 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 09a55dff35db417e99e0867c7c58cc02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731332180099 2024-11-11T13:36:23,303 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 978b1ac2546f46f2b72dabb1edb17c7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731332182262 2024-11-11T13:36:23,313 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/1bbb7a74c93349e0a10026920ca5ddbe as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1bbb7a74c93349e0a10026920ca5ddbe 2024-11-11T13:36:23,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:23,318 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/A of 7cb6411e18fb1479182e4865079a3fa6 into 1bbb7a74c93349e0a10026920ca5ddbe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:23,318 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:23,318 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/A, priority=13, startTime=1731332182824; duration=0sec 2024-11-11T13:36:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332243310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:23,318 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:23,318 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:A 2024-11-11T13:36:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:23,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332243310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:23,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:23,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332243310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:23,323 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#C#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:23,323 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/91bf58e4ba5144e2bbf283a312ff1f4e is 50, key is test_row_0/C:col10/1731332182262/Put/seqid=0 2024-11-11T13:36:23,331 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/6d226b6c6a0e4dc0a6bb4d363ef3fd5d 2024-11-11T13:36:23,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/c4519676f1aa46b6b648dfeeeb854408 is 50, key is test_row_0/B:col10/1731332182316/Put/seqid=0 2024-11-11T13:36:23,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742127_1303 (size=13119) 2024-11-11T13:36:23,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742128_1304 (size=12301) 2024-11-11T13:36:23,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332243629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:23,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:23,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332243629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:23,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:23,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332243629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T13:36:23,774 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/91bf58e4ba5144e2bbf283a312ff1f4e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/91bf58e4ba5144e2bbf283a312ff1f4e 2024-11-11T13:36:23,778 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/C of 7cb6411e18fb1479182e4865079a3fa6 into 91bf58e4ba5144e2bbf283a312ff1f4e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:23,778 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:23,778 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/C, priority=13, startTime=1731332182824; duration=0sec 2024-11-11T13:36:23,778 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:23,778 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:C 2024-11-11T13:36:23,806 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/c4519676f1aa46b6b648dfeeeb854408 2024-11-11T13:36:23,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/f94bc68012dc4cf0bff3c9d3894b31d3 is 50, key is test_row_0/C:col10/1731332182316/Put/seqid=0 2024-11-11T13:36:23,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742129_1305 (size=12301) 2024-11-11T13:36:23,820 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/f94bc68012dc4cf0bff3c9d3894b31d3 2024-11-11T13:36:23,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/6d226b6c6a0e4dc0a6bb4d363ef3fd5d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/6d226b6c6a0e4dc0a6bb4d363ef3fd5d 2024-11-11T13:36:23,831 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/6d226b6c6a0e4dc0a6bb4d363ef3fd5d, entries=150, sequenceid=369, filesize=12.0 K 2024-11-11T13:36:23,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/c4519676f1aa46b6b648dfeeeb854408 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c4519676f1aa46b6b648dfeeeb854408 2024-11-11T13:36:23,838 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c4519676f1aa46b6b648dfeeeb854408, entries=150, sequenceid=369, filesize=12.0 K 2024-11-11T13:36:23,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/f94bc68012dc4cf0bff3c9d3894b31d3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f94bc68012dc4cf0bff3c9d3894b31d3 2024-11-11T13:36:23,848 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f94bc68012dc4cf0bff3c9d3894b31d3, entries=150, sequenceid=369, filesize=12.0 K 2024-11-11T13:36:23,849 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7cb6411e18fb1479182e4865079a3fa6 in 962ms, sequenceid=369, compaction requested=false 2024-11-11T13:36:23,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:23,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:23,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-11T13:36:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-11T13:36:23,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T13:36:23,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2870 sec 2024-11-11T13:36:23,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.2920 sec 2024-11-11T13:36:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:24,141 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:24,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T13:36:24,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/8a9e8494c40f4077aa5a8bdb0cc08507 is 50, key is test_row_0/A:col10/1731332184140/Put/seqid=0 2024-11-11T13:36:24,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742130_1306 (size=14741) 2024-11-11T13:36:24,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/8a9e8494c40f4077aa5a8bdb0cc08507 2024-11-11T13:36:24,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/f3b270f8b14a4feb9bd57aafd45add72 is 50, key is test_row_0/B:col10/1731332184140/Put/seqid=0 2024-11-11T13:36:24,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742131_1307 (size=12301) 2024-11-11T13:36:24,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332244181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332244182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332244188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332244290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332244291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332244299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43322 deadline: 1731332244469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,471 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in 7cb6411e18fb1479182e4865079a3fa6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43334 deadline: 1731332244479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,480 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in 7cb6411e18fb1479182e4865079a3fa6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18309 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332244499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332244499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332244509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/f3b270f8b14a4feb9bd57aafd45add72 2024-11-11T13:36:24,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/4155ce4bb99c4844aa53b6b57e5488c8 is 50, key is test_row_0/C:col10/1731332184140/Put/seqid=0 2024-11-11T13:36:24,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742132_1308 (size=12301) 2024-11-11T13:36:24,608 INFO [Thread-1080 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1080. 2024-11-11T13:36:24,608 DEBUG [Thread-1080 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:24,609 DEBUG [Thread-1080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:24,609 DEBUG [Thread-1080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:24,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:24,610 INFO [Thread-1078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1078. 2024-11-11T13:36:24,610 DEBUG [Thread-1078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:24,610 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:24,610 DEBUG [Thread-1078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:24,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:24,611 INFO [Thread-1086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1086. 2024-11-11T13:36:24,611 DEBUG [Thread-1086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:24,611 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:24,611 INFO [Thread-1084 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1084. 2024-11-11T13:36:24,611 DEBUG [Thread-1084 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:24,611 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:24,611 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:24,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:24,612 DEBUG [Thread-1084 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:24,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:24,614 INFO [Thread-1082 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1082. 2024-11-11T13:36:24,614 DEBUG [Thread-1082 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:24,614 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:24,614 DEBUG [Thread-1082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:24,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T13:36:24,720 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:24,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43356 deadline: 1731332244810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43352 deadline: 1731332244810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43380 deadline: 1731332244819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:24,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:36:24,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/4155ce4bb99c4844aa53b6b57e5488c8 2024-11-11T13:36:24,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/8a9e8494c40f4077aa5a8bdb0cc08507 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/8a9e8494c40f4077aa5a8bdb0cc08507 2024-11-11T13:36:24,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/8a9e8494c40f4077aa5a8bdb0cc08507, entries=200, sequenceid=387, filesize=14.4 K 2024-11-11T13:36:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/f3b270f8b14a4feb9bd57aafd45add72 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f3b270f8b14a4feb9bd57aafd45add72 2024-11-11T13:36:24,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f3b270f8b14a4feb9bd57aafd45add72, entries=150, sequenceid=387, filesize=12.0 K 2024-11-11T13:36:24,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/4155ce4bb99c4844aa53b6b57e5488c8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/4155ce4bb99c4844aa53b6b57e5488c8 2024-11-11T13:36:25,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/4155ce4bb99c4844aa53b6b57e5488c8, entries=150, sequenceid=387, filesize=12.0 K 2024-11-11T13:36:25,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 7cb6411e18fb1479182e4865079a3fa6 in 860ms, sequenceid=387, compaction requested=true 2024-11-11T13:36:25,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:25,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:25,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:25,002 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:25,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:25,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:25,002 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:25,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7cb6411e18fb1479182e4865079a3fa6:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:25,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:25,002 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:25,002 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:25,002 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/A is initiating minor compaction (all files) 2024-11-11T13:36:25,002 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/B is initiating minor compaction (all files) 2024-11-11T13:36:25,003 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/A in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:25,003 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/B in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:25,003 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1bbb7a74c93349e0a10026920ca5ddbe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/6d226b6c6a0e4dc0a6bb4d363ef3fd5d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/8a9e8494c40f4077aa5a8bdb0cc08507] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=39.2 K 2024-11-11T13:36:25,003 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/87e702c8ece0477b97a95e586c938803, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c4519676f1aa46b6b648dfeeeb854408, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f3b270f8b14a4feb9bd57aafd45add72] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=36.8 K 2024-11-11T13:36:25,003 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bbb7a74c93349e0a10026920ca5ddbe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731332182262 2024-11-11T13:36:25,003 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 87e702c8ece0477b97a95e586c938803, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731332182262 2024-11-11T13:36:25,003 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d226b6c6a0e4dc0a6bb4d363ef3fd5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731332182310 2024-11-11T13:36:25,003 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c4519676f1aa46b6b648dfeeeb854408, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731332182310 2024-11-11T13:36:25,003 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f3b270f8b14a4feb9bd57aafd45add72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731332182979 2024-11-11T13:36:25,003 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a9e8494c40f4077aa5a8bdb0cc08507, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731332182979 2024-11-11T13:36:25,010 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:25,010 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:25,010 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/db58e35bc04442c38b55aa759bf519a6 is 50, key is test_row_0/B:col10/1731332184140/Put/seqid=0 2024-11-11T13:36:25,010 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/b150f77478bb4810b2952efab4dd7e27 is 50, key is test_row_0/A:col10/1731332184140/Put/seqid=0 2024-11-11T13:36:25,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742134_1310 (size=13221) 2024-11-11T13:36:25,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742133_1309 (size=13221) 2024-11-11T13:36:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:25,321 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:25,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T13:36:25,321 INFO [Thread-1075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1075. 2024-11-11T13:36:25,321 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-11-11T13:36:25,321 DEBUG [Thread-1075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:25,321 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:25,321 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:25,321 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:25,321 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:25,321 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:25,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:25,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:25,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/4614afd819fe4fb28ec31aa37918c9f9 is 50, key is test_row_0/A:col10/1731332184168/Put/seqid=0 2024-11-11T13:36:25,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742135_1311 (size=12301) 2024-11-11T13:36:25,331 INFO [Thread-1067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1067. 2024-11-11T13:36:25,331 DEBUG [Thread-1067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:25,331 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:25,331 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:25,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:25,422 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/b150f77478bb4810b2952efab4dd7e27 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b150f77478bb4810b2952efab4dd7e27 2024-11-11T13:36:25,422 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/db58e35bc04442c38b55aa759bf519a6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/db58e35bc04442c38b55aa759bf519a6 2024-11-11T13:36:25,428 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/A of 7cb6411e18fb1479182e4865079a3fa6 into b150f77478bb4810b2952efab4dd7e27(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:25,428 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/B of 7cb6411e18fb1479182e4865079a3fa6 into db58e35bc04442c38b55aa759bf519a6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:25,428 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:25,428 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:25,428 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/A, priority=13, startTime=1731332185002; duration=0sec 2024-11-11T13:36:25,428 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/B, priority=13, startTime=1731332185002; duration=0sec 2024-11-11T13:36:25,428 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:25,428 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:A 2024-11-11T13:36:25,428 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:25,428 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:B 2024-11-11T13:36:25,428 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:25,429 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:25,429 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 7cb6411e18fb1479182e4865079a3fa6/C is initiating minor compaction (all files) 2024-11-11T13:36:25,430 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7cb6411e18fb1479182e4865079a3fa6/C in TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:25,430 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/91bf58e4ba5144e2bbf283a312ff1f4e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f94bc68012dc4cf0bff3c9d3894b31d3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/4155ce4bb99c4844aa53b6b57e5488c8] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp, totalSize=36.8 K 2024-11-11T13:36:25,430 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91bf58e4ba5144e2bbf283a312ff1f4e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731332182262 2024-11-11T13:36:25,430 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f94bc68012dc4cf0bff3c9d3894b31d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731332182310 2024-11-11T13:36:25,431 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4155ce4bb99c4844aa53b6b57e5488c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731332182979 2024-11-11T13:36:25,437 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7cb6411e18fb1479182e4865079a3fa6#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:25,437 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/f3aeae4371b648e8a1bbdfd572e9addc is 50, key is test_row_0/C:col10/1731332184140/Put/seqid=0 2024-11-11T13:36:25,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742136_1312 (size=13221) 2024-11-11T13:36:25,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/4614afd819fe4fb28ec31aa37918c9f9 2024-11-11T13:36:25,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/0a458ff6416443e7ab41e3a888ef3cfd is 50, key is test_row_0/B:col10/1731332184168/Put/seqid=0 2024-11-11T13:36:25,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742137_1313 (size=12301) 2024-11-11T13:36:25,846 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/f3aeae4371b648e8a1bbdfd572e9addc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f3aeae4371b648e8a1bbdfd572e9addc 2024-11-11T13:36:25,850 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7cb6411e18fb1479182e4865079a3fa6/C of 7cb6411e18fb1479182e4865079a3fa6 into f3aeae4371b648e8a1bbdfd572e9addc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:25,850 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:25,850 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., storeName=7cb6411e18fb1479182e4865079a3fa6/C, priority=13, startTime=1731332185002; duration=0sec 2024-11-11T13:36:25,850 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:25,850 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7cb6411e18fb1479182e4865079a3fa6:C 2024-11-11T13:36:26,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/0a458ff6416443e7ab41e3a888ef3cfd 2024-11-11T13:36:26,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/f998b2a569764978bc28e45e7a84fe39 is 50, key is test_row_0/C:col10/1731332184168/Put/seqid=0 2024-11-11T13:36:26,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742138_1314 (size=12301) 2024-11-11T13:36:26,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/f998b2a569764978bc28e45e7a84fe39 2024-11-11T13:36:26,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/4614afd819fe4fb28ec31aa37918c9f9 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/4614afd819fe4fb28ec31aa37918c9f9 2024-11-11T13:36:26,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/4614afd819fe4fb28ec31aa37918c9f9, entries=150, sequenceid=410, filesize=12.0 K 2024-11-11T13:36:26,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/0a458ff6416443e7ab41e3a888ef3cfd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/0a458ff6416443e7ab41e3a888ef3cfd 2024-11-11T13:36:26,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/0a458ff6416443e7ab41e3a888ef3cfd, entries=150, sequenceid=410, filesize=12.0 K 2024-11-11T13:36:26,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/f998b2a569764978bc28e45e7a84fe39 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f998b2a569764978bc28e45e7a84fe39 2024-11-11T13:36:26,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f998b2a569764978bc28e45e7a84fe39, entries=150, sequenceid=410, filesize=12.0 K 2024-11-11T13:36:26,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=6.71 KB/6870 for 7cb6411e18fb1479182e4865079a3fa6 in 1263ms, sequenceid=410, compaction requested=false 2024-11-11T13:36:26,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7cb6411e18fb1479182e4865079a3fa6: 2024-11-11T13:36:29,757 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T13:36:34,531 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-11-11T13:36:34,531 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:34,531 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:34,531 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:34,531 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:34,541 INFO [Thread-1073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1073. 2024-11-11T13:36:34,541 DEBUG [Thread-1073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:34,541 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:34,541 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:34,541 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:34,541 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T13:36:34,541 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-11T13:36:34,541 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-11T13:36:34,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-11T13:36:34,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-11T13:36:34,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-11T13:36:34,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T13:36:34,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3729 2024-11-11T13:36:34,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3683 2024-11-11T13:36:34,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3511 2024-11-11T13:36:34,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3687 2024-11-11T13:36:34,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3718 2024-11-11T13:36:34,542 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T13:36:34,542 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:36:34,542 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:36:34,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:34,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:34,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:34,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T13:36:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T13:36:34,546 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332194546"}]},"ts":"1731332194546"} 2024-11-11T13:36:34,548 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T13:36:34,548 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T13:36:34,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T13:36:34,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cb6411e18fb1479182e4865079a3fa6, UNASSIGN}] 2024-11-11T13:36:34,552 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cb6411e18fb1479182e4865079a3fa6, UNASSIGN 2024-11-11T13:36:34,553 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=7cb6411e18fb1479182e4865079a3fa6, regionState=CLOSING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:34,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cb6411e18fb1479182e4865079a3fa6, UNASSIGN because future has completed 2024-11-11T13:36:34,556 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T13:36:34,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:36:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T13:36:34,709 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(122): Close 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:34,709 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T13:36:34,709 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1722): Closing 7cb6411e18fb1479182e4865079a3fa6, disabling compactions & flushes 2024-11-11T13:36:34,709 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:34,709 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:34,709 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. after waiting 0 ms 2024-11-11T13:36:34,709 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:34,709 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(2902): Flushing 7cb6411e18fb1479182e4865079a3fa6 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-11T13:36:34,714 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/f9a237ba92354d8fbced24f4d9ec917e is 50, key is test_row_0/A:col10/1731332194540/Put/seqid=0 2024-11-11T13:36:34,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742139_1315 (size=9857) 2024-11-11T13:36:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T13:36:35,120 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/f9a237ba92354d8fbced24f4d9ec917e 2024-11-11T13:36:35,127 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/8a3d4a8fa9d74892bdfef0114bfe0926 is 50, key is test_row_0/B:col10/1731332194540/Put/seqid=0 2024-11-11T13:36:35,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742140_1316 (size=9857) 2024-11-11T13:36:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T13:36:35,533 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/8a3d4a8fa9d74892bdfef0114bfe0926 2024-11-11T13:36:35,539 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/3d7ebd58ef9841f29ed47e4137e18c3a is 50, key is test_row_0/C:col10/1731332194540/Put/seqid=0 2024-11-11T13:36:35,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742141_1317 (size=9857) 2024-11-11T13:36:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T13:36:35,944 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/3d7ebd58ef9841f29ed47e4137e18c3a 2024-11-11T13:36:35,949 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/A/f9a237ba92354d8fbced24f4d9ec917e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/f9a237ba92354d8fbced24f4d9ec917e 2024-11-11T13:36:35,953 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/f9a237ba92354d8fbced24f4d9ec917e, entries=100, sequenceid=419, filesize=9.6 K 2024-11-11T13:36:35,954 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/B/8a3d4a8fa9d74892bdfef0114bfe0926 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8a3d4a8fa9d74892bdfef0114bfe0926 2024-11-11T13:36:35,957 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8a3d4a8fa9d74892bdfef0114bfe0926, entries=100, sequenceid=419, filesize=9.6 K 2024-11-11T13:36:35,958 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/.tmp/C/3d7ebd58ef9841f29ed47e4137e18c3a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/3d7ebd58ef9841f29ed47e4137e18c3a 2024-11-11T13:36:35,961 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/3d7ebd58ef9841f29ed47e4137e18c3a, entries=100, sequenceid=419, filesize=9.6 K 2024-11-11T13:36:35,961 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 7cb6411e18fb1479182e4865079a3fa6 in 1252ms, sequenceid=419, compaction requested=true 2024-11-11T13:36:35,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/a8dea4d5ff8542fb891ea6d805ead2e7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7092ed912da04329bb109412e197899a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/a48044f21e35401fb1752ad39fe98660, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7ca56fc8d3d34c08b1b6933a417b6323, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/5336facef3a647838f8b7357e364ec4a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e42f96287c4dc0b2ef1f9981902904, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/e9730c1177be453f9251ad70ff59fde8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e40b63a8ca46b7b602acfffa14ec1f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/bd4b85b618b24811b6107b8bf499d449, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/2f4afb64d1b4459f8618ec75314260a1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b7204c293efe4603a1246c97ab727934, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/085fa76e781f4a3cbdee12d3050e665b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/6d88699357eb4e9093fe4862725c9984, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/97a088fd249b43ce883de8e02c33528a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c423620a05e4820a96677d144a6f1d0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c9cb5922c4d4de7a9f80da5fa841ef0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/24b25a59f5464072b176ff3c4250e55b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/d51d5687b4124d8b827fa02f01effbea, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b0f8af95ea5c49f894706dc95a841fe8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/f85dbb0443864280b56f084f36f6c3f8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/bc7cc84eca6a41f5ac5d0edddead290f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/3a3882dfb7c44a29a1c0b325e2f781df, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/fbe44a1905d74a08913874123f52918a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1bbb7a74c93349e0a10026920ca5ddbe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/540ce077c8a54a53bdca5cb5673d7f78, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/6d226b6c6a0e4dc0a6bb4d363ef3fd5d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/8a9e8494c40f4077aa5a8bdb0cc08507] to archive 2024-11-11T13:36:35,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:36:35,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/a8dea4d5ff8542fb891ea6d805ead2e7 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/a8dea4d5ff8542fb891ea6d805ead2e7 2024-11-11T13:36:35,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7092ed912da04329bb109412e197899a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7092ed912da04329bb109412e197899a 2024-11-11T13:36:35,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/a48044f21e35401fb1752ad39fe98660 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/a48044f21e35401fb1752ad39fe98660 2024-11-11T13:36:35,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7ca56fc8d3d34c08b1b6933a417b6323 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/7ca56fc8d3d34c08b1b6933a417b6323 2024-11-11T13:36:35,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/5336facef3a647838f8b7357e364ec4a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/5336facef3a647838f8b7357e364ec4a 2024-11-11T13:36:35,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e42f96287c4dc0b2ef1f9981902904 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e42f96287c4dc0b2ef1f9981902904 2024-11-11T13:36:35,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/e9730c1177be453f9251ad70ff59fde8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/e9730c1177be453f9251ad70ff59fde8 2024-11-11T13:36:35,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e40b63a8ca46b7b602acfffa14ec1f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/c3e40b63a8ca46b7b602acfffa14ec1f 2024-11-11T13:36:35,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/bd4b85b618b24811b6107b8bf499d449 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/bd4b85b618b24811b6107b8bf499d449 2024-11-11T13:36:35,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/2f4afb64d1b4459f8618ec75314260a1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/2f4afb64d1b4459f8618ec75314260a1 2024-11-11T13:36:35,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b7204c293efe4603a1246c97ab727934 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b7204c293efe4603a1246c97ab727934 2024-11-11T13:36:35,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/085fa76e781f4a3cbdee12d3050e665b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/085fa76e781f4a3cbdee12d3050e665b 2024-11-11T13:36:35,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/6d88699357eb4e9093fe4862725c9984 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/6d88699357eb4e9093fe4862725c9984 2024-11-11T13:36:35,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/97a088fd249b43ce883de8e02c33528a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/97a088fd249b43ce883de8e02c33528a 2024-11-11T13:36:35,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c423620a05e4820a96677d144a6f1d0 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c423620a05e4820a96677d144a6f1d0 2024-11-11T13:36:35,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c9cb5922c4d4de7a9f80da5fa841ef0 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1c9cb5922c4d4de7a9f80da5fa841ef0 2024-11-11T13:36:35,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/24b25a59f5464072b176ff3c4250e55b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/24b25a59f5464072b176ff3c4250e55b 2024-11-11T13:36:35,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/d51d5687b4124d8b827fa02f01effbea to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/d51d5687b4124d8b827fa02f01effbea 2024-11-11T13:36:35,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b0f8af95ea5c49f894706dc95a841fe8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b0f8af95ea5c49f894706dc95a841fe8 2024-11-11T13:36:35,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/f85dbb0443864280b56f084f36f6c3f8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/f85dbb0443864280b56f084f36f6c3f8 2024-11-11T13:36:35,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/bc7cc84eca6a41f5ac5d0edddead290f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/bc7cc84eca6a41f5ac5d0edddead290f 2024-11-11T13:36:35,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/3a3882dfb7c44a29a1c0b325e2f781df to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/3a3882dfb7c44a29a1c0b325e2f781df 2024-11-11T13:36:35,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/fbe44a1905d74a08913874123f52918a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/fbe44a1905d74a08913874123f52918a 2024-11-11T13:36:35,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1bbb7a74c93349e0a10026920ca5ddbe to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/1bbb7a74c93349e0a10026920ca5ddbe 2024-11-11T13:36:35,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/540ce077c8a54a53bdca5cb5673d7f78 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/540ce077c8a54a53bdca5cb5673d7f78 2024-11-11T13:36:35,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/6d226b6c6a0e4dc0a6bb4d363ef3fd5d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/6d226b6c6a0e4dc0a6bb4d363ef3fd5d 2024-11-11T13:36:35,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/8a9e8494c40f4077aa5a8bdb0cc08507 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/8a9e8494c40f4077aa5a8bdb0cc08507 2024-11-11T13:36:35,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/2ac919496308476b9afe93930d185c7d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/9cc8648708b74a199d0cee2c2ae13727, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f9f4a86ee2b34d27801735dfa1ae8a0d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/42b69ab86e124cd3bfb89318d8897bec, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/085275ef10a148e6a04008db093d7aa0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/52fc7fab10954878bd8a541a1bcc1bc5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/df770b3eb35943f89a4b9c46811f452d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/cbbd813b8113475889f141a9d1e85cb2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/739d8071fe134059b81a057f53487f74, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/7aaec148c9234d1e99a757c4ed0151cc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c3587d313f614e80b6b8cf16236418f1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/a0492661f00d4620ac0ed9d470e9e210, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/ac923184b7cf46ada0afbc4927b96c2b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/60c2fa24a6e6454dbad55b01cf74163e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/541e631eb22244f4b262a6e7c6177e9a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/894b53bec6ed4ef68112dc2e70e84273, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/78bc9f1f7523469f82a184cd16831fea, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f672333bd7614ac1bb625d5e7e50de3d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8d80c072b8754621afc3273360ccd2fd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/4e0d0170ed27428db2430a1b2df1a9bd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/0f8fc739b6b6432b8fb8bf83f8b97046, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8dd53baaf7ee4574af1ee780c7491f38, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/89b1c6ddcd4a4b4c8a67bf16becfee62, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/87e702c8ece0477b97a95e586c938803, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/91e0c45e258845c095c54bd81cfeda60, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c4519676f1aa46b6b648dfeeeb854408, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f3b270f8b14a4feb9bd57aafd45add72] to archive 2024-11-11T13:36:35,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:36:35,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/2ac919496308476b9afe93930d185c7d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/2ac919496308476b9afe93930d185c7d 2024-11-11T13:36:35,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/9cc8648708b74a199d0cee2c2ae13727 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/9cc8648708b74a199d0cee2c2ae13727 2024-11-11T13:36:35,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f9f4a86ee2b34d27801735dfa1ae8a0d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f9f4a86ee2b34d27801735dfa1ae8a0d 2024-11-11T13:36:35,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/42b69ab86e124cd3bfb89318d8897bec to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/42b69ab86e124cd3bfb89318d8897bec 2024-11-11T13:36:35,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/085275ef10a148e6a04008db093d7aa0 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/085275ef10a148e6a04008db093d7aa0 2024-11-11T13:36:35,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/52fc7fab10954878bd8a541a1bcc1bc5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/52fc7fab10954878bd8a541a1bcc1bc5 2024-11-11T13:36:35,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/df770b3eb35943f89a4b9c46811f452d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/df770b3eb35943f89a4b9c46811f452d 2024-11-11T13:36:36,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/cbbd813b8113475889f141a9d1e85cb2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/cbbd813b8113475889f141a9d1e85cb2 2024-11-11T13:36:36,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/739d8071fe134059b81a057f53487f74 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/739d8071fe134059b81a057f53487f74 2024-11-11T13:36:36,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/7aaec148c9234d1e99a757c4ed0151cc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/7aaec148c9234d1e99a757c4ed0151cc 2024-11-11T13:36:36,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c3587d313f614e80b6b8cf16236418f1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c3587d313f614e80b6b8cf16236418f1 2024-11-11T13:36:36,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/a0492661f00d4620ac0ed9d470e9e210 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/a0492661f00d4620ac0ed9d470e9e210 2024-11-11T13:36:36,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/ac923184b7cf46ada0afbc4927b96c2b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/ac923184b7cf46ada0afbc4927b96c2b 2024-11-11T13:36:36,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/60c2fa24a6e6454dbad55b01cf74163e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/60c2fa24a6e6454dbad55b01cf74163e 2024-11-11T13:36:36,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/541e631eb22244f4b262a6e7c6177e9a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/541e631eb22244f4b262a6e7c6177e9a 2024-11-11T13:36:36,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/894b53bec6ed4ef68112dc2e70e84273 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/894b53bec6ed4ef68112dc2e70e84273 2024-11-11T13:36:36,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/78bc9f1f7523469f82a184cd16831fea to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/78bc9f1f7523469f82a184cd16831fea 2024-11-11T13:36:36,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f672333bd7614ac1bb625d5e7e50de3d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f672333bd7614ac1bb625d5e7e50de3d 2024-11-11T13:36:36,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8d80c072b8754621afc3273360ccd2fd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8d80c072b8754621afc3273360ccd2fd 2024-11-11T13:36:36,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/4e0d0170ed27428db2430a1b2df1a9bd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/4e0d0170ed27428db2430a1b2df1a9bd 2024-11-11T13:36:36,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/0f8fc739b6b6432b8fb8bf83f8b97046 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/0f8fc739b6b6432b8fb8bf83f8b97046 2024-11-11T13:36:36,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8dd53baaf7ee4574af1ee780c7491f38 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8dd53baaf7ee4574af1ee780c7491f38 2024-11-11T13:36:36,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/89b1c6ddcd4a4b4c8a67bf16becfee62 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/89b1c6ddcd4a4b4c8a67bf16becfee62 2024-11-11T13:36:36,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/87e702c8ece0477b97a95e586c938803 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/87e702c8ece0477b97a95e586c938803 2024-11-11T13:36:36,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/91e0c45e258845c095c54bd81cfeda60 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/91e0c45e258845c095c54bd81cfeda60 2024-11-11T13:36:36,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c4519676f1aa46b6b648dfeeeb854408 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/c4519676f1aa46b6b648dfeeeb854408 2024-11-11T13:36:36,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f3b270f8b14a4feb9bd57aafd45add72 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/f3b270f8b14a4feb9bd57aafd45add72 2024-11-11T13:36:36,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1baf5c8e9b684c448f58be9785b3018a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/0cfc47a2ae674b8c8dd700e9acd6bc64, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d0cd7bac37c64437aadbce78c458ee64, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1e9ffd84803d471da3e430718901cc60, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/3812e5a8d91949c19e55fc346c43496a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/2e6fc01a34d24fce837015e9d515b6bb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/13743ca702e34b7cb3be06adfd452a7b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9e7d16ae403c49e98b74ab0ae0c0139b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9723c5bade5948c79e8616dc48f1c218, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/012c448e32ee4f5a98dca834bbaef3a1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f61df8584f0f4b95822325feaa29dd43, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/23a8c5a520ef4527b56dc448cc32236a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/ed51c1dd1dac4cbdb009248f7d12c532, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d42011ad50ff4f858623b54a3ff40351, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1a885556a5d043068155f12378729d9a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1ea9fcfb715040dab8fd058948d88ad9, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/cb04f8f6e70d4eb2b1ba942232753623, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/8eb79cdc2aa34a16a7ec7d407d1d9de9, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/adfed80e95a94fc9afc62d3e58dee8dd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d5caac23bc0047b3abb845518d05c9e4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/401fd369255240808245becd5d02fe15, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/69f6feb999ce4d758a4cd5a617b81a6e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/09a55dff35db417e99e0867c7c58cc02, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/91bf58e4ba5144e2bbf283a312ff1f4e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/978b1ac2546f46f2b72dabb1edb17c7a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f94bc68012dc4cf0bff3c9d3894b31d3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/4155ce4bb99c4844aa53b6b57e5488c8] to archive 2024-11-11T13:36:36,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:36:36,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1baf5c8e9b684c448f58be9785b3018a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1baf5c8e9b684c448f58be9785b3018a 2024-11-11T13:36:36,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/0cfc47a2ae674b8c8dd700e9acd6bc64 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/0cfc47a2ae674b8c8dd700e9acd6bc64 2024-11-11T13:36:36,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d0cd7bac37c64437aadbce78c458ee64 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d0cd7bac37c64437aadbce78c458ee64 2024-11-11T13:36:36,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1e9ffd84803d471da3e430718901cc60 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1e9ffd84803d471da3e430718901cc60 2024-11-11T13:36:36,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/3812e5a8d91949c19e55fc346c43496a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/3812e5a8d91949c19e55fc346c43496a 2024-11-11T13:36:36,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/2e6fc01a34d24fce837015e9d515b6bb to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/2e6fc01a34d24fce837015e9d515b6bb 2024-11-11T13:36:36,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/13743ca702e34b7cb3be06adfd452a7b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/13743ca702e34b7cb3be06adfd452a7b 2024-11-11T13:36:36,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9e7d16ae403c49e98b74ab0ae0c0139b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9e7d16ae403c49e98b74ab0ae0c0139b 2024-11-11T13:36:36,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9723c5bade5948c79e8616dc48f1c218 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/9723c5bade5948c79e8616dc48f1c218 2024-11-11T13:36:36,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/012c448e32ee4f5a98dca834bbaef3a1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/012c448e32ee4f5a98dca834bbaef3a1 2024-11-11T13:36:36,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f61df8584f0f4b95822325feaa29dd43 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f61df8584f0f4b95822325feaa29dd43 2024-11-11T13:36:36,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/23a8c5a520ef4527b56dc448cc32236a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/23a8c5a520ef4527b56dc448cc32236a 2024-11-11T13:36:36,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/ed51c1dd1dac4cbdb009248f7d12c532 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/ed51c1dd1dac4cbdb009248f7d12c532 2024-11-11T13:36:36,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d42011ad50ff4f858623b54a3ff40351 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d42011ad50ff4f858623b54a3ff40351 2024-11-11T13:36:36,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1a885556a5d043068155f12378729d9a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1a885556a5d043068155f12378729d9a 2024-11-11T13:36:36,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1ea9fcfb715040dab8fd058948d88ad9 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/1ea9fcfb715040dab8fd058948d88ad9 2024-11-11T13:36:36,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/cb04f8f6e70d4eb2b1ba942232753623 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/cb04f8f6e70d4eb2b1ba942232753623 2024-11-11T13:36:36,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/8eb79cdc2aa34a16a7ec7d407d1d9de9 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/8eb79cdc2aa34a16a7ec7d407d1d9de9 2024-11-11T13:36:36,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/adfed80e95a94fc9afc62d3e58dee8dd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/adfed80e95a94fc9afc62d3e58dee8dd 2024-11-11T13:36:36,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d5caac23bc0047b3abb845518d05c9e4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/d5caac23bc0047b3abb845518d05c9e4 2024-11-11T13:36:36,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/401fd369255240808245becd5d02fe15 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/401fd369255240808245becd5d02fe15 2024-11-11T13:36:36,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/69f6feb999ce4d758a4cd5a617b81a6e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/69f6feb999ce4d758a4cd5a617b81a6e 2024-11-11T13:36:36,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/09a55dff35db417e99e0867c7c58cc02 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/09a55dff35db417e99e0867c7c58cc02 2024-11-11T13:36:36,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/91bf58e4ba5144e2bbf283a312ff1f4e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/91bf58e4ba5144e2bbf283a312ff1f4e 2024-11-11T13:36:36,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/978b1ac2546f46f2b72dabb1edb17c7a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/978b1ac2546f46f2b72dabb1edb17c7a 2024-11-11T13:36:36,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f94bc68012dc4cf0bff3c9d3894b31d3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f94bc68012dc4cf0bff3c9d3894b31d3 2024-11-11T13:36:36,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/4155ce4bb99c4844aa53b6b57e5488c8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/4155ce4bb99c4844aa53b6b57e5488c8 2024-11-11T13:36:36,063 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/recovered.edits/422.seqid, newMaxSeqId=422, maxSeqId=1 2024-11-11T13:36:36,063 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. 2024-11-11T13:36:36,063 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1676): Region close journal for 7cb6411e18fb1479182e4865079a3fa6: Waiting for close lock at 1731332194709Running coprocessor pre-close hooks at 1731332194709Disabling compacts and flushes for region at 1731332194709Disabling writes for close at 1731332194709Obtaining lock to block concurrent updates at 1731332194709Preparing flush snapshotting stores in 7cb6411e18fb1479182e4865079a3fa6 at 1731332194709Finished memstore snapshotting TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731332194710 (+1 ms)Flushing stores of TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6. at 1731332194710Flushing 7cb6411e18fb1479182e4865079a3fa6/A: creating writer at 1731332194710Flushing 7cb6411e18fb1479182e4865079a3fa6/A: appending metadata at 1731332194714 (+4 ms)Flushing 7cb6411e18fb1479182e4865079a3fa6/A: closing flushed file at 1731332194714Flushing 7cb6411e18fb1479182e4865079a3fa6/B: creating writer at 1731332195124 (+410 ms)Flushing 7cb6411e18fb1479182e4865079a3fa6/B: appending metadata at 1731332195126 (+2 ms)Flushing 7cb6411e18fb1479182e4865079a3fa6/B: closing flushed file at 1731332195126Flushing 7cb6411e18fb1479182e4865079a3fa6/C: creating writer at 1731332195537 (+411 ms)Flushing 7cb6411e18fb1479182e4865079a3fa6/C: appending metadata at 1731332195539 (+2 ms)Flushing 7cb6411e18fb1479182e4865079a3fa6/C: closing flushed file at 1731332195539Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64e1cc80: reopening flushed file at 1731332195948 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37f763e8: reopening flushed file at 1731332195953 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4205f439: reopening flushed file at 1731332195957 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 7cb6411e18fb1479182e4865079a3fa6 in 1252ms, sequenceid=419, compaction requested=true at 1731332195961 (+4 ms)Writing region close event to WAL at 1731332196060 (+99 ms)Running coprocessor post-close hooks at 1731332196063 (+3 ms)Closed at 1731332196063 2024-11-11T13:36:36,065 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(157): Closed 7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:36,066 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=7cb6411e18fb1479182e4865079a3fa6, regionState=CLOSED 2024-11-11T13:36:36,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:36:36,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T13:36:36,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseRegionProcedure 7cb6411e18fb1479182e4865079a3fa6, server=37d26ecf6460,36877,1731332101576 in 1.5120 sec 2024-11-11T13:36:36,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-11T13:36:36,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7cb6411e18fb1479182e4865079a3fa6, UNASSIGN in 1.5180 sec 2024-11-11T13:36:36,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T13:36:36,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5220 sec 2024-11-11T13:36:36,074 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332196074"}]},"ts":"1731332196074"} 2024-11-11T13:36:36,075 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T13:36:36,075 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T13:36:36,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5330 sec 2024-11-11T13:36:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T13:36:36,691 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:36,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T13:36:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:36,721 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T13:36:36,723 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:36,727 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:36,729 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/recovered.edits] 2024-11-11T13:36:36,731 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/4614afd819fe4fb28ec31aa37918c9f9 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/4614afd819fe4fb28ec31aa37918c9f9 2024-11-11T13:36:36,732 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b150f77478bb4810b2952efab4dd7e27 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/b150f77478bb4810b2952efab4dd7e27 2024-11-11T13:36:36,733 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/f9a237ba92354d8fbced24f4d9ec917e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/A/f9a237ba92354d8fbced24f4d9ec917e 2024-11-11T13:36:36,734 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/0a458ff6416443e7ab41e3a888ef3cfd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/0a458ff6416443e7ab41e3a888ef3cfd 2024-11-11T13:36:36,735 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8a3d4a8fa9d74892bdfef0114bfe0926 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/8a3d4a8fa9d74892bdfef0114bfe0926 2024-11-11T13:36:36,736 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/db58e35bc04442c38b55aa759bf519a6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/B/db58e35bc04442c38b55aa759bf519a6 2024-11-11T13:36:36,738 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/3d7ebd58ef9841f29ed47e4137e18c3a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/3d7ebd58ef9841f29ed47e4137e18c3a 2024-11-11T13:36:36,738 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f3aeae4371b648e8a1bbdfd572e9addc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f3aeae4371b648e8a1bbdfd572e9addc 2024-11-11T13:36:36,739 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f998b2a569764978bc28e45e7a84fe39 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/C/f998b2a569764978bc28e45e7a84fe39 2024-11-11T13:36:36,742 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/recovered.edits/422.seqid to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6/recovered.edits/422.seqid 2024-11-11T13:36:36,743 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/7cb6411e18fb1479182e4865079a3fa6 2024-11-11T13:36:36,743 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T13:36:36,745 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:36,751 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T13:36:36,753 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T13:36:36,754 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:36,754 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T13:36:36,755 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731332196754"}]},"ts":"9223372036854775807"} 2024-11-11T13:36:36,756 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T13:36:36,756 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7cb6411e18fb1479182e4865079a3fa6, NAME => 'TestAcidGuarantees,,1731332162402.7cb6411e18fb1479182e4865079a3fa6.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T13:36:36,756 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T13:36:36,756 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731332196756"}]},"ts":"9223372036854775807"} 2024-11-11T13:36:36,758 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T13:36:36,759 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:36,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-11T13:36:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T13:36:36,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T13:36:36,830 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:36,838 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=236 (was 239), OpenFileDescriptor=445 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=622 (was 499) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2916 (was 2929) 2024-11-11T13:36:36,847 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=236, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=622, ProcessCount=11, AvailableMemoryMB=2904 2024-11-11T13:36:36,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T13:36:36,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:36:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:36,850 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T13:36:36,850 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:36,851 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T13:36:36,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 79 2024-11-11T13:36:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-11T13:36:36,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742142_1318 (size=959) 2024-11-11T13:36:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-11T13:36:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-11T13:36:37,259 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92 2024-11-11T13:36:37,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742143_1319 (size=53) 2024-11-11T13:36:37,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:36:37,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0e1cf571f2e70bf460349ad52744187e, disabling compactions & flushes 2024-11-11T13:36:37,266 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:37,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:37,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. after waiting 0 ms 2024-11-11T13:36:37,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:37,266 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:37,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0e1cf571f2e70bf460349ad52744187e: Waiting for close lock at 1731332197266Disabling compacts and flushes for region at 1731332197266Disabling writes for close at 1731332197266Writing region close event to WAL at 1731332197266Closed at 1731332197266 2024-11-11T13:36:37,267 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T13:36:37,268 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731332197268"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731332197268"}]},"ts":"1731332197268"} 2024-11-11T13:36:37,270 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T13:36:37,271 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T13:36:37,271 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332197271"}]},"ts":"1731332197271"} 2024-11-11T13:36:37,273 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T13:36:37,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, ASSIGN}] 2024-11-11T13:36:37,276 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, ASSIGN 2024-11-11T13:36:37,276 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, ASSIGN; state=OFFLINE, location=37d26ecf6460,36877,1731332101576; forceNewPlan=false, retain=false 2024-11-11T13:36:37,428 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=0e1cf571f2e70bf460349ad52744187e, regionState=OPENING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:37,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, ASSIGN because future has completed 2024-11-11T13:36:37,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:36:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-11T13:36:37,591 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:37,591 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:36:37,592 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,592 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:36:37,592 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,592 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,595 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,598 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e1cf571f2e70bf460349ad52744187e columnFamilyName A 2024-11-11T13:36:37,598 DEBUG [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:37,599 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(327): Store=0e1cf571f2e70bf460349ad52744187e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:36:37,599 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,600 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e1cf571f2e70bf460349ad52744187e columnFamilyName B 2024-11-11T13:36:37,600 DEBUG [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:37,601 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(327): Store=0e1cf571f2e70bf460349ad52744187e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:36:37,601 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,601 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e1cf571f2e70bf460349ad52744187e columnFamilyName C 2024-11-11T13:36:37,601 DEBUG [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:37,602 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(327): Store=0e1cf571f2e70bf460349ad52744187e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:36:37,602 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,602 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,603 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,604 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,604 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,604 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T13:36:37,605 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,607 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:36:37,607 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 0e1cf571f2e70bf460349ad52744187e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60369927, jitterRate=-0.10041798651218414}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T13:36:37,607 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:37,608 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 0e1cf571f2e70bf460349ad52744187e: Running coprocessor pre-open hook at 1731332197593Writing region info on filesystem at 1731332197593Initializing all the Stores at 1731332197594 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332197594Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332197595 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332197595Cleaning up temporary data from old regions at 1731332197604 (+9 ms)Running coprocessor post-open hooks at 1731332197607 (+3 ms)Region opened successfully at 1731332197607 2024-11-11T13:36:37,608 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., pid=81, masterSystemTime=1731332197584 2024-11-11T13:36:37,610 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:37,610 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:37,610 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=0e1cf571f2e70bf460349ad52744187e, regionState=OPEN, openSeqNum=2, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:37,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:36:37,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T13:36:37,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 in 181 msec 2024-11-11T13:36:37,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-11T13:36:37,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, ASSIGN in 340 msec 2024-11-11T13:36:37,617 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T13:36:37,617 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332197617"}]},"ts":"1731332197617"} 2024-11-11T13:36:37,618 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T13:36:37,619 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=79, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T13:36:37,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 771 msec 2024-11-11T13:36:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-11-11T13:36:37,990 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:37,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60504243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:37,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:37,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:37,992 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:37,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:37,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f7d0ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:37,994 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f8c20c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:37,995 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:37,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:37,998 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T13:36:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 37d26ecf6460,35609,1731332100707 2024-11-11T13:36:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@11d532da 2024-11-11T13:36:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T13:36:37,999 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T13:36:38,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T13:36:38,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:36:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T13:36:38,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742144_1320 (size=995) 2024-11-11T13:36:38,413 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T13:36:38,414 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T13:36:38,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T13:36:38,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, REOPEN/MOVE}] 2024-11-11T13:36:38,422 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, REOPEN/MOVE 2024-11-11T13:36:38,423 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=0e1cf571f2e70bf460349ad52744187e, regionState=CLOSING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:38,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, REOPEN/MOVE because future has completed 2024-11-11T13:36:38,424 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T13:36:38,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:36:38,577 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,577 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T13:36:38,577 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 0e1cf571f2e70bf460349ad52744187e, disabling compactions & flushes 2024-11-11T13:36:38,577 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:38,577 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:38,577 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. after waiting 0 ms 2024-11-11T13:36:38,577 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:38,581 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T13:36:38,581 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:38,581 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 0e1cf571f2e70bf460349ad52744187e: Waiting for close lock at 1731332198577Running coprocessor pre-close hooks at 1731332198577Disabling compacts and flushes for region at 1731332198577Disabling writes for close at 1731332198577Writing region close event to WAL at 1731332198577Running coprocessor post-close hooks at 1731332198581 (+4 ms)Closed at 1731332198581 2024-11-11T13:36:38,581 WARN [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionServer(3299): Not adding moved region record: 0e1cf571f2e70bf460349ad52744187e to self. 2024-11-11T13:36:38,583 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,583 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=0e1cf571f2e70bf460349ad52744187e, regionState=CLOSED 2024-11-11T13:36:38,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:36:38,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-11T13:36:38,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 in 161 msec 2024-11-11T13:36:38,588 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, REOPEN/MOVE; state=CLOSED, location=37d26ecf6460,36877,1731332101576; forceNewPlan=false, retain=true 2024-11-11T13:36:38,738 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=0e1cf571f2e70bf460349ad52744187e, regionState=OPENING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:38,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, REOPEN/MOVE because future has completed 2024-11-11T13:36:38,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:36:38,894 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:38,894 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:36:38,895 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,895 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:36:38,895 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7794): checking encryption for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,895 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(7797): checking classloading for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,896 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,897 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e1cf571f2e70bf460349ad52744187e columnFamilyName A 2024-11-11T13:36:38,898 DEBUG [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:38,899 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(327): Store=0e1cf571f2e70bf460349ad52744187e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:36:38,899 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,900 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e1cf571f2e70bf460349ad52744187e columnFamilyName B 2024-11-11T13:36:38,900 DEBUG [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:38,900 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(327): Store=0e1cf571f2e70bf460349ad52744187e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:36:38,900 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,901 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e1cf571f2e70bf460349ad52744187e columnFamilyName C 2024-11-11T13:36:38,901 DEBUG [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:38,901 INFO [StoreOpener-0e1cf571f2e70bf460349ad52744187e-1 {}] regionserver.HStore(327): Store=0e1cf571f2e70bf460349ad52744187e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:36:38,901 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1038): replaying wal for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,902 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,903 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,904 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1048): stopping wal replay for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,904 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1060): Cleaning up temporary data for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,904 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T13:36:38,905 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1093): writing seq id for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,906 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1114): Opened 0e1cf571f2e70bf460349ad52744187e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63831561, jitterRate=-0.048835620284080505}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T13:36:38,906 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:38,907 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegion(1006): Region open journal for 0e1cf571f2e70bf460349ad52744187e: Running coprocessor pre-open hook at 1731332198895Writing region info on filesystem at 1731332198895Initializing all the Stores at 1731332198896 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332198896Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332198896Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332198896Cleaning up temporary data from old regions at 1731332198904 (+8 ms)Running coprocessor post-open hooks at 1731332198906 (+2 ms)Region opened successfully at 1731332198906 2024-11-11T13:36:38,907 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., pid=86, masterSystemTime=1731332198891 2024-11-11T13:36:38,909 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:38,909 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=86}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:38,909 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=0e1cf571f2e70bf460349ad52744187e, regionState=OPEN, openSeqNum=5, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:38,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:36:38,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=84 2024-11-11T13:36:38,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 in 171 msec 2024-11-11T13:36:38,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-11T13:36:38,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, REOPEN/MOVE in 491 msec 2024-11-11T13:36:38,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-11T13:36:38,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 496 msec 2024-11-11T13:36:38,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-11-11T13:36:38,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T13:36:38,920 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:38,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b7cc840, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:38,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:38,923 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1c61c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:38,925 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f43faf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b7442bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:38,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:38,929 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9b5d631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:38,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:38,931 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:38,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed1aad6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b7a602a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,935 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:38,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:38,937 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dbb42e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:38,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:38,938 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75cf1b67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b08f19b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:38,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:38,942 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@debb59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:38,944 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:38,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd45bb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a9b5f49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:38,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:38,949 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b54c43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:38,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:38,953 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f27f90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebdf278, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,954 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:38,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:38,956 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3060a03e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:38,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:38,959 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e812db8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2654a436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:38,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:38,965 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@210537a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:38,967 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d35f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63172990, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,968 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:38,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:38,969 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5832b9f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:38,971 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8bb93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213b5311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:38,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:38,975 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4734ee8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:38,985 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:38,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10aaa8d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e1f600, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:36:38,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:36:38,989 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:36:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:36:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:36:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e35ae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:36:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:36:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:38,991 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:36:38,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14210673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:36:38,995 DEBUG [Thread-1479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:38,995 DEBUG [Thread-1475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:38,995 DEBUG [Thread-1477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:38,996 DEBUG [Thread-1482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:38,996 DEBUG [Thread-1477 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:38,997 DEBUG [Thread-1484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:38,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:38,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:38,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:38,998 DEBUG [Thread-1471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:38,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:38,999 DEBUG [Thread-1473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:39,000 DEBUG [Thread-1477 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:39,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:39,001 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:39,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:36:39,002 DEBUG [Thread-1473 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:39,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:39,002 DEBUG [Thread-1473 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:39,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:39,003 DEBUG [Thread-1490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:39,004 DEBUG [Thread-1488 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:39,004 DEBUG [Thread-1486 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:36:39,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:39,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:36:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:36:39,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:36:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:36:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:36:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:39,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-11T13:36:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T13:36:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:36:39,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:36:39,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:36:39,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:36:39,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:36:39,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:36:39,026 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:39,027 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:39,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:36:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:39,045 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:39,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:36:39,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111d839b2e92a84a1c99ef74edb1005f48_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332199044/Put/seqid=0 2024-11-11T13:36:39,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742145_1321 (size=12154) 2024-11-11T13:36:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T13:36:39,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332259137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332259137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332259147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332259148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332259150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:39,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:39,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332259259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332259259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332259259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332259259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332259259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T13:36:39,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:39,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:39,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332259469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332259469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332259469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332259470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332259470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:39,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:39,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,508 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:39,512 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111d839b2e92a84a1c99ef74edb1005f48_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111d839b2e92a84a1c99ef74edb1005f48_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:39,513 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/8835673a4e334132afeddf1f2ebd5c1c, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:39,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/8835673a4e334132afeddf1f2ebd5c1c is 175, key is test_row_0/A:col10/1731332199044/Put/seqid=0 2024-11-11T13:36:39,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742146_1322 (size=30955) 2024-11-11T13:36:39,537 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/8835673a4e334132afeddf1f2ebd5c1c 2024-11-11T13:36:39,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/3d9e39b8926b47f8961dd6687ffafb7b is 50, key is test_row_0/B:col10/1731332199044/Put/seqid=0 2024-11-11T13:36:39,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742147_1323 (size=12001) 2024-11-11T13:36:39,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/3d9e39b8926b47f8961dd6687ffafb7b 2024-11-11T13:36:39,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/95055f29a6524892b9a3034854e07f65 is 50, key is test_row_0/C:col10/1731332199044/Put/seqid=0 2024-11-11T13:36:39,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742148_1324 (size=12001) 2024-11-11T13:36:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T13:36:39,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/95055f29a6524892b9a3034854e07f65 2024-11-11T13:36:39,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:39,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/8835673a4e334132afeddf1f2ebd5c1c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/8835673a4e334132afeddf1f2ebd5c1c 2024-11-11T13:36:39,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:39,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/8835673a4e334132afeddf1f2ebd5c1c, entries=150, sequenceid=16, filesize=30.2 K 2024-11-11T13:36:39,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/3d9e39b8926b47f8961dd6687ffafb7b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/3d9e39b8926b47f8961dd6687ffafb7b 2024-11-11T13:36:39,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/3d9e39b8926b47f8961dd6687ffafb7b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T13:36:39,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/95055f29a6524892b9a3034854e07f65 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95055f29a6524892b9a3034854e07f65 2024-11-11T13:36:39,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95055f29a6524892b9a3034854e07f65, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T13:36:39,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0e1cf571f2e70bf460349ad52744187e in 625ms, sequenceid=16, compaction requested=false 2024-11-11T13:36:39,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:39,783 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:39,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:36:39,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113dbdfd080e1e4b8ca91e0d3cf0c62fbb_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332199783/Put/seqid=0 2024-11-11T13:36:39,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332259790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332259792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332259794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332259795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332259796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742149_1325 (size=17034) 2024-11-11T13:36:39,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332259909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332259909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332259910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332259910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332259914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:39,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:39,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:39,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:39,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:40,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:40,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332260129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332260130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332260136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332260136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332260136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T13:36:40,213 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:40,256 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113dbdfd080e1e4b8ca91e0d3cf0c62fbb_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113dbdfd080e1e4b8ca91e0d3cf0c62fbb_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:40,267 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/325c27114ee243df9dea2f2f52ad8e66, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:40,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/325c27114ee243df9dea2f2f52ad8e66 is 175, key is test_row_0/A:col10/1731332199783/Put/seqid=0 2024-11-11T13:36:40,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:40,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:40,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742150_1326 (size=48139) 2024-11-11T13:36:40,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:40,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:40,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332260449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332260450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332260450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332260450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332260450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:40,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:40,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,718 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/325c27114ee243df9dea2f2f52ad8e66 2024-11-11T13:36:40,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/83ae8d8f6a094209a1a7c2afc3f16402 is 50, key is test_row_0/B:col10/1731332199783/Put/seqid=0 2024-11-11T13:36:40,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742151_1327 (size=12001) 2024-11-11T13:36:40,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/83ae8d8f6a094209a1a7c2afc3f16402 2024-11-11T13:36:40,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/60e358f9d2784675b4c5c64f9e716393 is 50, key is test_row_0/C:col10/1731332199783/Put/seqid=0 2024-11-11T13:36:40,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:40,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:40,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742152_1328 (size=12001) 2024-11-11T13:36:40,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:40,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:40,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:40,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332260959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332260959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332260959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332260959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332260959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:41,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:41,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:41,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:41,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:41,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:41,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:41,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/60e358f9d2784675b4c5c64f9e716393 2024-11-11T13:36:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T13:36:41,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/325c27114ee243df9dea2f2f52ad8e66 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/325c27114ee243df9dea2f2f52ad8e66 2024-11-11T13:36:41,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/325c27114ee243df9dea2f2f52ad8e66, entries=250, sequenceid=42, filesize=47.0 K 2024-11-11T13:36:41,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/83ae8d8f6a094209a1a7c2afc3f16402 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/83ae8d8f6a094209a1a7c2afc3f16402 2024-11-11T13:36:41,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/83ae8d8f6a094209a1a7c2afc3f16402, entries=150, sequenceid=42, filesize=11.7 K 2024-11-11T13:36:41,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/60e358f9d2784675b4c5c64f9e716393 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/60e358f9d2784675b4c5c64f9e716393 2024-11-11T13:36:41,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/60e358f9d2784675b4c5c64f9e716393, entries=150, sequenceid=42, filesize=11.7 K 2024-11-11T13:36:41,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0e1cf571f2e70bf460349ad52744187e in 1411ms, sequenceid=42, compaction requested=false 2024-11-11T13:36:41,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:41,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T13:36:41,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:41,211 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-11T13:36:41,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f8177faac1424444b49ffc08711b45f9_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332199794/Put/seqid=0 2024-11-11T13:36:41,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742153_1329 (size=12154) 2024-11-11T13:36:41,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:41,318 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f8177faac1424444b49ffc08711b45f9_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f8177faac1424444b49ffc08711b45f9_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:41,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/f6696617db694689a81fa64b9c2cc2c4, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:41,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/f6696617db694689a81fa64b9c2cc2c4 is 175, key is test_row_0/A:col10/1731332199794/Put/seqid=0 2024-11-11T13:36:41,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742154_1330 (size=30955) 2024-11-11T13:36:41,609 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T13:36:41,787 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/f6696617db694689a81fa64b9c2cc2c4 2024-11-11T13:36:41,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/852b0a180e4442808a5dc1c613a9591a is 50, key is test_row_0/B:col10/1731332199794/Put/seqid=0 2024-11-11T13:36:41,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742155_1331 (size=12001) 2024-11-11T13:36:41,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:42,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332262052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332262052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332262053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332262062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332262072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332262179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332262180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332262180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332262180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,199 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/852b0a180e4442808a5dc1c613a9591a 2024-11-11T13:36:42,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332262189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/0e60d6e170c248849dccd64abc5fcdc8 is 50, key is test_row_0/C:col10/1731332199794/Put/seqid=0 2024-11-11T13:36:42,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742156_1332 (size=12001) 2024-11-11T13:36:42,213 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/0e60d6e170c248849dccd64abc5fcdc8 2024-11-11T13:36:42,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/f6696617db694689a81fa64b9c2cc2c4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/f6696617db694689a81fa64b9c2cc2c4 2024-11-11T13:36:42,234 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/f6696617db694689a81fa64b9c2cc2c4, entries=150, sequenceid=52, filesize=30.2 K 2024-11-11T13:36:42,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/852b0a180e4442808a5dc1c613a9591a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/852b0a180e4442808a5dc1c613a9591a 2024-11-11T13:36:42,240 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/852b0a180e4442808a5dc1c613a9591a, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T13:36:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/0e60d6e170c248849dccd64abc5fcdc8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/0e60d6e170c248849dccd64abc5fcdc8 2024-11-11T13:36:42,277 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/0e60d6e170c248849dccd64abc5fcdc8, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T13:36:42,280 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 0e1cf571f2e70bf460349ad52744187e in 1068ms, sequenceid=52, compaction requested=true 2024-11-11T13:36:42,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:42,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:42,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-11T13:36:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-11T13:36:42,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-11T13:36:42,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2550 sec 2024-11-11T13:36:42,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 3.2830 sec 2024-11-11T13:36:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:42,397 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:42,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-11T13:36:42,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111122f050aeaaca4a28950166e1cfae1212_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332202051/Put/seqid=0 2024-11-11T13:36:42,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332262413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332262414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332262415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332262423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332262424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742157_1333 (size=17034) 2024-11-11T13:36:42,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:42,485 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111122f050aeaaca4a28950166e1cfae1212_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111122f050aeaaca4a28950166e1cfae1212_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:42,491 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/7caf30cde76341d0a9e2ceaf2bafeb31, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:42,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/7caf30cde76341d0a9e2ceaf2bafeb31 is 175, key is test_row_0/A:col10/1731332202051/Put/seqid=0 2024-11-11T13:36:42,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742158_1334 (size=48139) 2024-11-11T13:36:42,524 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/7caf30cde76341d0a9e2ceaf2bafeb31 2024-11-11T13:36:42,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332262530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332262530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332262535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332262550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332262550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/685c6fbaa0964de98e969cfddc737391 is 50, key is test_row_0/B:col10/1731332202051/Put/seqid=0 2024-11-11T13:36:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742159_1335 (size=12001) 2024-11-11T13:36:42,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/685c6fbaa0964de98e969cfddc737391 2024-11-11T13:36:42,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/64c82e3afd9a47ad93ebc002175972bf is 50, key is test_row_0/C:col10/1731332202051/Put/seqid=0 2024-11-11T13:36:42,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742160_1336 (size=12001) 2024-11-11T13:36:42,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332262749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332262749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332262749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332262759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:42,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332262769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:42,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:42,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/64c82e3afd9a47ad93ebc002175972bf 2024-11-11T13:36:43,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/7caf30cde76341d0a9e2ceaf2bafeb31 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/7caf30cde76341d0a9e2ceaf2bafeb31 2024-11-11T13:36:43,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/7caf30cde76341d0a9e2ceaf2bafeb31, entries=250, sequenceid=80, filesize=47.0 K 2024-11-11T13:36:43,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/685c6fbaa0964de98e969cfddc737391 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/685c6fbaa0964de98e969cfddc737391 2024-11-11T13:36:43,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/685c6fbaa0964de98e969cfddc737391, entries=150, sequenceid=80, filesize=11.7 K 2024-11-11T13:36:43,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/64c82e3afd9a47ad93ebc002175972bf as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/64c82e3afd9a47ad93ebc002175972bf 2024-11-11T13:36:43,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/64c82e3afd9a47ad93ebc002175972bf, entries=150, sequenceid=80, filesize=11.7 K 2024-11-11T13:36:43,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 0e1cf571f2e70bf460349ad52744187e in 629ms, sequenceid=80, compaction requested=true 2024-11-11T13:36:43,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:43,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:43,027 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:43,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:43,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:43,027 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:43,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:43,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:43,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:43,028 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158188 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:43,028 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:43,028 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/A is initiating minor compaction (all files) 2024-11-11T13:36:43,028 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/B is initiating minor compaction (all files) 2024-11-11T13:36:43,028 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/B in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:43,028 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/A in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:43,028 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/3d9e39b8926b47f8961dd6687ffafb7b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/83ae8d8f6a094209a1a7c2afc3f16402, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/852b0a180e4442808a5dc1c613a9591a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/685c6fbaa0964de98e969cfddc737391] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=46.9 K 2024-11-11T13:36:43,028 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/8835673a4e334132afeddf1f2ebd5c1c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/325c27114ee243df9dea2f2f52ad8e66, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/f6696617db694689a81fa64b9c2cc2c4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/7caf30cde76341d0a9e2ceaf2bafeb31] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=154.5 K 2024-11-11T13:36:43,028 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:43,028 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/8835673a4e334132afeddf1f2ebd5c1c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/325c27114ee243df9dea2f2f52ad8e66, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/f6696617db694689a81fa64b9c2cc2c4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/7caf30cde76341d0a9e2ceaf2bafeb31] 2024-11-11T13:36:43,028 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d9e39b8926b47f8961dd6687ffafb7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731332199032 2024-11-11T13:36:43,031 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 83ae8d8f6a094209a1a7c2afc3f16402, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731332199146 2024-11-11T13:36:43,031 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8835673a4e334132afeddf1f2ebd5c1c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731332199032 2024-11-11T13:36:43,031 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 852b0a180e4442808a5dc1c613a9591a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332199787 2024-11-11T13:36:43,031 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 325c27114ee243df9dea2f2f52ad8e66, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731332199146 2024-11-11T13:36:43,032 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 685c6fbaa0964de98e969cfddc737391, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731332202051 2024-11-11T13:36:43,032 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6696617db694689a81fa64b9c2cc2c4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332199787 2024-11-11T13:36:43,032 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7caf30cde76341d0a9e2ceaf2bafeb31, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731332202051 2024-11-11T13:36:43,056 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:43,064 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#B#compaction#288 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:43,065 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/375148c14e324000b131601cdedb4a1f is 50, key is test_row_0/B:col10/1731332202051/Put/seqid=0 2024-11-11T13:36:43,068 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111eb3e9204d73f4c9890045015b35ec8ad_0e1cf571f2e70bf460349ad52744187e store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:43,070 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:43,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:36:43,070 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111eb3e9204d73f4c9890045015b35ec8ad_0e1cf571f2e70bf460349ad52744187e, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:43,070 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111eb3e9204d73f4c9890045015b35ec8ad_0e1cf571f2e70bf460349ad52744187e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:43,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742161_1337 (size=12139) 2024-11-11T13:36:43,103 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/375148c14e324000b131601cdedb4a1f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/375148c14e324000b131601cdedb4a1f 2024-11-11T13:36:43,109 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/B of 0e1cf571f2e70bf460349ad52744187e into 375148c14e324000b131601cdedb4a1f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:43,109 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:43,109 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/B, priority=12, startTime=1731332203027; duration=0sec 2024-11-11T13:36:43,109 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:43,109 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:B 2024-11-11T13:36:43,109 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:43,111 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:43,111 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/C is initiating minor compaction (all files) 2024-11-11T13:36:43,111 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/C in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:43,111 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95055f29a6524892b9a3034854e07f65, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/60e358f9d2784675b4c5c64f9e716393, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/0e60d6e170c248849dccd64abc5fcdc8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/64c82e3afd9a47ad93ebc002175972bf] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=46.9 K 2024-11-11T13:36:43,111 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 95055f29a6524892b9a3034854e07f65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731332199032 2024-11-11T13:36:43,112 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e358f9d2784675b4c5c64f9e716393, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731332199146 2024-11-11T13:36:43,112 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e60d6e170c248849dccd64abc5fcdc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731332199787 2024-11-11T13:36:43,112 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 64c82e3afd9a47ad93ebc002175972bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731332202051 2024-11-11T13:36:43,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118101e4f78ff64cdba0b5f3039c2381b9_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332202398/Put/seqid=0 2024-11-11T13:36:43,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742163_1339 (size=17034) 2024-11-11T13:36:43,128 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:43,133 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118101e4f78ff64cdba0b5f3039c2381b9_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118101e4f78ff64cdba0b5f3039c2381b9_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:43,134 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/ead3a9c0a4fc4f8ab5f6810133402dca, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:43,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/ead3a9c0a4fc4f8ab5f6810133402dca is 175, key is test_row_0/A:col10/1731332202398/Put/seqid=0 2024-11-11T13:36:43,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742162_1338 (size=4469) 2024-11-11T13:36:43,139 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#C#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:43,140 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/e240066dcdc947ce99b07f2545038b4d is 50, key is test_row_0/C:col10/1731332202051/Put/seqid=0 2024-11-11T13:36:43,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742164_1340 (size=48139) 2024-11-11T13:36:43,143 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/ead3a9c0a4fc4f8ab5f6810133402dca 2024-11-11T13:36:43,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742165_1341 (size=12139) 2024-11-11T13:36:43,151 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/e240066dcdc947ce99b07f2545038b4d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e240066dcdc947ce99b07f2545038b4d 2024-11-11T13:36:43,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/866627ee5e8f484993e7a9b6389da37e is 50, key is test_row_0/B:col10/1731332202398/Put/seqid=0 2024-11-11T13:36:43,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742166_1342 (size=12001) 2024-11-11T13:36:43,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/866627ee5e8f484993e7a9b6389da37e 2024-11-11T13:36:43,165 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/C of 0e1cf571f2e70bf460349ad52744187e into e240066dcdc947ce99b07f2545038b4d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:43,165 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:43,166 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/C, priority=12, startTime=1731332203027; duration=0sec 2024-11-11T13:36:43,166 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:43,166 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:C 2024-11-11T13:36:43,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332263150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332263151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332263152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332263154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332263156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/78edfc0cf76d4654a080aaf5e81896a6 is 50, key is test_row_0/C:col10/1731332202398/Put/seqid=0 2024-11-11T13:36:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T13:36:43,184 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:43,187 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-11T13:36:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T13:36:43,193 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:43,194 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:43,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:43,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742167_1343 (size=12001) 2024-11-11T13:36:43,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332263279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332263279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332263280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332263280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332263280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T13:36:43,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-11T13:36:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:43,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:43,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:43,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332263494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-11T13:36:43,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:43,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:43,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:43,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:43,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:43,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332263500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332263500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332263503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332263503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T13:36:43,543 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#A#compaction#289 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:43,544 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/089b5ab3c07745958dbed6fdee27c511 is 175, key is test_row_0/A:col10/1731332202051/Put/seqid=0 2024-11-11T13:36:43,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742168_1344 (size=31093) 2024-11-11T13:36:43,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/78edfc0cf76d4654a080aaf5e81896a6 2024-11-11T13:36:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/ead3a9c0a4fc4f8ab5f6810133402dca as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/ead3a9c0a4fc4f8ab5f6810133402dca 2024-11-11T13:36:43,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/ead3a9c0a4fc4f8ab5f6810133402dca, entries=250, sequenceid=91, filesize=47.0 K 2024-11-11T13:36:43,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/866627ee5e8f484993e7a9b6389da37e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/866627ee5e8f484993e7a9b6389da37e 2024-11-11T13:36:43,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/866627ee5e8f484993e7a9b6389da37e, entries=150, sequenceid=91, filesize=11.7 K 2024-11-11T13:36:43,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/78edfc0cf76d4654a080aaf5e81896a6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/78edfc0cf76d4654a080aaf5e81896a6 2024-11-11T13:36:43,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/78edfc0cf76d4654a080aaf5e81896a6, entries=150, sequenceid=91, filesize=11.7 K 2024-11-11T13:36:43,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0e1cf571f2e70bf460349ad52744187e in 551ms, sequenceid=91, compaction requested=false 2024-11-11T13:36:43,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:43,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-11T13:36:43,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:43,659 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:36:43,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cf6b3faac47f4a26b60cc05445d14bd7_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332203152/Put/seqid=0 2024-11-11T13:36:43,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742169_1345 (size=12154) 2024-11-11T13:36:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:43,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:43,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332263824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332263824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332263825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332263826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332263829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T13:36:43,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332263939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332263939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332263940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332263940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:43,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332263940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:43,992 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/089b5ab3c07745958dbed6fdee27c511 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/089b5ab3c07745958dbed6fdee27c511 2024-11-11T13:36:44,005 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/A of 0e1cf571f2e70bf460349ad52744187e into 089b5ab3c07745958dbed6fdee27c511(size=30.4 K), total size for store is 77.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:44,005 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:44,005 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/A, priority=12, startTime=1731332203026; duration=0sec 2024-11-11T13:36:44,005 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:44,005 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:A 2024-11-11T13:36:44,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:44,093 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cf6b3faac47f4a26b60cc05445d14bd7_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cf6b3faac47f4a26b60cc05445d14bd7_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:44,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/4360b96d0fb3474e818eb37bbda79b7c, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:44,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/4360b96d0fb3474e818eb37bbda79b7c is 175, key is test_row_0/A:col10/1731332203152/Put/seqid=0 2024-11-11T13:36:44,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742170_1346 (size=30955) 2024-11-11T13:36:44,142 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/4360b96d0fb3474e818eb37bbda79b7c 2024-11-11T13:36:44,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332264149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332264149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332264150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332264150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332264151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/576a2a0f7dee45c2b57b4c8de624c664 is 50, key is test_row_0/B:col10/1731332203152/Put/seqid=0 2024-11-11T13:36:44,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742171_1347 (size=12001) 2024-11-11T13:36:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T13:36:44,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332264470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332264470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332264471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332264472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332264472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,591 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/576a2a0f7dee45c2b57b4c8de624c664 2024-11-11T13:36:44,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/ac6ee2b81454460899ce129ea69cff19 is 50, key is test_row_0/C:col10/1731332203152/Put/seqid=0 2024-11-11T13:36:44,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742172_1348 (size=12001) 2024-11-11T13:36:44,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332264979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332264980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332264980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332264989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:44,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332264989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:45,021 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/ac6ee2b81454460899ce129ea69cff19 2024-11-11T13:36:45,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/4360b96d0fb3474e818eb37bbda79b7c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/4360b96d0fb3474e818eb37bbda79b7c 2024-11-11T13:36:45,042 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/4360b96d0fb3474e818eb37bbda79b7c, entries=150, sequenceid=118, filesize=30.2 K 2024-11-11T13:36:45,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/576a2a0f7dee45c2b57b4c8de624c664 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/576a2a0f7dee45c2b57b4c8de624c664 2024-11-11T13:36:45,049 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/576a2a0f7dee45c2b57b4c8de624c664, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T13:36:45,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/ac6ee2b81454460899ce129ea69cff19 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/ac6ee2b81454460899ce129ea69cff19 2024-11-11T13:36:45,061 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/ac6ee2b81454460899ce129ea69cff19, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T13:36:45,063 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0e1cf571f2e70bf460349ad52744187e in 1404ms, sequenceid=118, compaction requested=true 2024-11-11T13:36:45,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:45,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:45,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-11T13:36:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-11T13:36:45,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-11T13:36:45,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8770 sec 2024-11-11T13:36:45,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.8860 sec 2024-11-11T13:36:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T13:36:45,360 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:45,361 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-11T13:36:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T13:36:45,364 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:45,365 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:45,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T13:36:45,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T13:36:45,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:45,520 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:36:45,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116ece6801e28040818c570c7eeab5e7d2_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332203824/Put/seqid=0 2024-11-11T13:36:45,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742173_1349 (size=12154) 2024-11-11T13:36:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:45,662 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116ece6801e28040818c570c7eeab5e7d2_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116ece6801e28040818c570c7eeab5e7d2_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:45,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d5b47a95e9bf47b1b932d998933bcf60, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:45,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d5b47a95e9bf47b1b932d998933bcf60 is 175, key is test_row_0/A:col10/1731332203824/Put/seqid=0 2024-11-11T13:36:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T13:36:45,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742174_1350 (size=30955) 2024-11-11T13:36:45,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:45,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T13:36:46,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332266036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332266042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332266043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332266046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332266047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,093 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d5b47a95e9bf47b1b932d998933bcf60 2024-11-11T13:36:46,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/484aaf131cd642ba953c1beb20ec6816 is 50, key is test_row_0/B:col10/1731332203824/Put/seqid=0 2024-11-11T13:36:46,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742175_1351 (size=12001) 2024-11-11T13:36:46,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332266149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332266149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332266160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332266160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332266160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332266359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332266369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332266379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332266379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332266380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,504 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/484aaf131cd642ba953c1beb20ec6816 2024-11-11T13:36:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T13:36:46,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/7ef61101550d4c0ca30cbe6acc50fc75 is 50, key is test_row_0/C:col10/1731332203824/Put/seqid=0 2024-11-11T13:36:46,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742176_1352 (size=12001) 2024-11-11T13:36:46,514 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/7ef61101550d4c0ca30cbe6acc50fc75 2024-11-11T13:36:46,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d5b47a95e9bf47b1b932d998933bcf60 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5b47a95e9bf47b1b932d998933bcf60 2024-11-11T13:36:46,526 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5b47a95e9bf47b1b932d998933bcf60, entries=150, sequenceid=130, filesize=30.2 K 2024-11-11T13:36:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/484aaf131cd642ba953c1beb20ec6816 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/484aaf131cd642ba953c1beb20ec6816 2024-11-11T13:36:46,536 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/484aaf131cd642ba953c1beb20ec6816, entries=150, sequenceid=130, filesize=11.7 K 2024-11-11T13:36:46,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/7ef61101550d4c0ca30cbe6acc50fc75 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/7ef61101550d4c0ca30cbe6acc50fc75 2024-11-11T13:36:46,552 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/7ef61101550d4c0ca30cbe6acc50fc75, entries=150, sequenceid=130, filesize=11.7 K 2024-11-11T13:36:46,553 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0e1cf571f2e70bf460349ad52744187e in 1034ms, sequenceid=130, compaction requested=true 2024-11-11T13:36:46,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:46,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:46,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-11T13:36:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-11T13:36:46,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-11T13:36:46,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1890 sec 2024-11-11T13:36:46,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.2000 sec 2024-11-11T13:36:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:46,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:46,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T13:36:46,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332266705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332266705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332266709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332266716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332266716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111163d74662695e4ec180b8635b168e2d0d_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332206675/Put/seqid=0 2024-11-11T13:36:46,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742177_1353 (size=14794) 2024-11-11T13:36:46,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332266819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332266839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332266840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332267029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:47,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332267049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332267049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:47,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,199 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:47,202 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111163d74662695e4ec180b8635b168e2d0d_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111163d74662695e4ec180b8635b168e2d0d_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:47,203 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/bbfc25537eb3417d8c546d359fcb4d1e, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:47,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/bbfc25537eb3417d8c546d359fcb4d1e is 175, key is test_row_0/A:col10/1731332206675/Put/seqid=0 2024-11-11T13:36:47,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742178_1354 (size=39749) 2024-11-11T13:36:47,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332267219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332267219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332267349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332267369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:47,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332267370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:47,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T13:36:47,519 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:47,520 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-11T13:36:47,522 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T13:36:47,524 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:47,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:47,616 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/bbfc25537eb3417d8c546d359fcb4d1e 2024-11-11T13:36:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T13:36:47,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:47,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:47,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:47,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:47,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:47,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:47,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/8e2f00e02d5a46ecb3ead36997e3a2c7 is 50, key is test_row_0/B:col10/1731332206675/Put/seqid=0 2024-11-11T13:36:47,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742179_1355 (size=12151) 2024-11-11T13:36:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T13:36:47,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:47,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:47,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:47,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:47,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:47,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:47,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332267869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332267889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332267889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:47,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:47,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:47,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:47,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:47,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:47,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:47,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:47,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/8e2f00e02d5a46ecb3ead36997e3a2c7 2024-11-11T13:36:48,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/95016652bcc34ee4912f46ea37ee0f68 is 50, key is test_row_0/C:col10/1731332206675/Put/seqid=0 2024-11-11T13:36:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T13:36:48,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742180_1356 (size=12151) 2024-11-11T13:36:48,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:48,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:48,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/95016652bcc34ee4912f46ea37ee0f68 2024-11-11T13:36:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/bbfc25537eb3417d8c546d359fcb4d1e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/bbfc25537eb3417d8c546d359fcb4d1e 2024-11-11T13:36:48,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/bbfc25537eb3417d8c546d359fcb4d1e, entries=200, sequenceid=156, filesize=38.8 K 2024-11-11T13:36:48,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/8e2f00e02d5a46ecb3ead36997e3a2c7 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8e2f00e02d5a46ecb3ead36997e3a2c7 2024-11-11T13:36:48,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8e2f00e02d5a46ecb3ead36997e3a2c7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T13:36:48,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/95016652bcc34ee4912f46ea37ee0f68 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95016652bcc34ee4912f46ea37ee0f68 2024-11-11T13:36:48,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95016652bcc34ee4912f46ea37ee0f68, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T13:36:48,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0e1cf571f2e70bf460349ad52744187e in 1493ms, sequenceid=156, compaction requested=true 2024-11-11T13:36:48,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:48,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:48,171 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T13:36:48,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:48,171 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T13:36:48,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:48,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:48,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:48,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:48,173 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 180891 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T13:36:48,173 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/A is initiating minor compaction (all files) 2024-11-11T13:36:48,173 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/A in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,174 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/089b5ab3c07745958dbed6fdee27c511, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/ead3a9c0a4fc4f8ab5f6810133402dca, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/4360b96d0fb3474e818eb37bbda79b7c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5b47a95e9bf47b1b932d998933bcf60, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/bbfc25537eb3417d8c546d359fcb4d1e] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=176.7 K 2024-11-11T13:36:48,174 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,174 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/089b5ab3c07745958dbed6fdee27c511, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/ead3a9c0a4fc4f8ab5f6810133402dca, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/4360b96d0fb3474e818eb37bbda79b7c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5b47a95e9bf47b1b932d998933bcf60, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/bbfc25537eb3417d8c546d359fcb4d1e] 2024-11-11T13:36:48,174 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60293 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T13:36:48,174 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/B is initiating minor compaction (all files) 2024-11-11T13:36:48,174 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 089b5ab3c07745958dbed6fdee27c511, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731332202051 2024-11-11T13:36:48,174 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/B in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,174 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/375148c14e324000b131601cdedb4a1f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/866627ee5e8f484993e7a9b6389da37e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/576a2a0f7dee45c2b57b4c8de624c664, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/484aaf131cd642ba953c1beb20ec6816, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8e2f00e02d5a46ecb3ead36997e3a2c7] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=58.9 K 2024-11-11T13:36:48,174 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting ead3a9c0a4fc4f8ab5f6810133402dca, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731332202398 2024-11-11T13:36:48,175 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 375148c14e324000b131601cdedb4a1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731332202051 2024-11-11T13:36:48,175 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4360b96d0fb3474e818eb37bbda79b7c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731332203131 2024-11-11T13:36:48,175 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 866627ee5e8f484993e7a9b6389da37e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731332202398 2024-11-11T13:36:48,175 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5b47a95e9bf47b1b932d998933bcf60, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731332203822 2024-11-11T13:36:48,175 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 576a2a0f7dee45c2b57b4c8de624c664, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731332203131 2024-11-11T13:36:48,175 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 484aaf131cd642ba953c1beb20ec6816, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731332203822 2024-11-11T13:36:48,176 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e2f00e02d5a46ecb3ead36997e3a2c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332206045 2024-11-11T13:36:48,177 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbfc25537eb3417d8c546d359fcb4d1e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332206034 2024-11-11T13:36:48,186 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:48,187 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:48,188 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/f91ad12b75b64cd8bb042bb3fa480f08 is 50, key is test_row_0/B:col10/1731332206675/Put/seqid=0 2024-11-11T13:36:48,192 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111d70729d15b754ba7a5081026b1ebc0b0_0e1cf571f2e70bf460349ad52744187e store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:48,194 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111d70729d15b754ba7a5081026b1ebc0b0_0e1cf571f2e70bf460349ad52744187e, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:48,195 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d70729d15b754ba7a5081026b1ebc0b0_0e1cf571f2e70bf460349ad52744187e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:48,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742181_1357 (size=12459) 2024-11-11T13:36:48,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742182_1358 (size=4469) 2024-11-11T13:36:48,245 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#A#compaction#304 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:48,245 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d03a0ac63d254c5f92c175debe48b13e is 175, key is test_row_0/A:col10/1731332206675/Put/seqid=0 2024-11-11T13:36:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:48,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:48,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:36:48,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742183_1359 (size=31413) 2024-11-11T13:36:48,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113bd7fa7118464699a43b5e78179479e9_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332208248/Put/seqid=0 2024-11-11T13:36:48,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742184_1360 (size=14794) 2024-11-11T13:36:48,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:48,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:48,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332268415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:48,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332268418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:48,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332268529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:48,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332268529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:48,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,630 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/f91ad12b75b64cd8bb042bb3fa480f08 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/f91ad12b75b64cd8bb042bb3fa480f08 2024-11-11T13:36:48,645 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/B of 0e1cf571f2e70bf460349ad52744187e into f91ad12b75b64cd8bb042bb3fa480f08(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:48,645 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:48,645 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/B, priority=11, startTime=1731332208171; duration=0sec 2024-11-11T13:36:48,645 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:48,645 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:B 2024-11-11T13:36:48,645 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T13:36:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T13:36:48,664 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60293 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T13:36:48,664 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/C is initiating minor compaction (all files) 2024-11-11T13:36:48,664 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/C in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,664 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e240066dcdc947ce99b07f2545038b4d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/78edfc0cf76d4654a080aaf5e81896a6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/ac6ee2b81454460899ce129ea69cff19, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/7ef61101550d4c0ca30cbe6acc50fc75, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95016652bcc34ee4912f46ea37ee0f68] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=58.9 K 2024-11-11T13:36:48,665 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e240066dcdc947ce99b07f2545038b4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731332202051 2024-11-11T13:36:48,665 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 78edfc0cf76d4654a080aaf5e81896a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731332202398 2024-11-11T13:36:48,666 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting ac6ee2b81454460899ce129ea69cff19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731332203131 2024-11-11T13:36:48,666 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ef61101550d4c0ca30cbe6acc50fc75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731332203822 2024-11-11T13:36:48,667 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 95016652bcc34ee4912f46ea37ee0f68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332206045 2024-11-11T13:36:48,669 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d03a0ac63d254c5f92c175debe48b13e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d03a0ac63d254c5f92c175debe48b13e 2024-11-11T13:36:48,682 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#C#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:48,683 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/4498b617b2784352a03a7645df1027cb is 50, key is test_row_0/C:col10/1731332206675/Put/seqid=0 2024-11-11T13:36:48,693 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/A of 0e1cf571f2e70bf460349ad52744187e into d03a0ac63d254c5f92c175debe48b13e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:48,693 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:48,693 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/A, priority=11, startTime=1731332208170; duration=0sec 2024-11-11T13:36:48,693 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:48,693 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:A 2024-11-11T13:36:48,698 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:48,716 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113bd7fa7118464699a43b5e78179479e9_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113bd7fa7118464699a43b5e78179479e9_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:48,717 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d5886b21ae674c77b9b287e954d94397, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:48,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d5886b21ae674c77b9b287e954d94397 is 175, key is test_row_0/A:col10/1731332208248/Put/seqid=0 2024-11-11T13:36:48,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742185_1361 (size=12459) 2024-11-11T13:36:48,749 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/4498b617b2784352a03a7645df1027cb as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/4498b617b2784352a03a7645df1027cb 2024-11-11T13:36:48,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742186_1362 (size=39749) 2024-11-11T13:36:48,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332268742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:48,761 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/C of 0e1cf571f2e70bf460349ad52744187e into 4498b617b2784352a03a7645df1027cb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,761 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:48,761 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/C, priority=11, startTime=1731332208171; duration=0sec 2024-11-11T13:36:48,761 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:48,761 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:C 2024-11-11T13:36:48,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332268751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:48,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:48,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:48,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332268889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:48,893 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_2' in 0e1cf571f2e70bf460349ad52744187e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:48,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332268909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:48,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:48,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332268909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:48,911 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in 0e1cf571f2e70bf460349ad52744187e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:48,911 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in 0e1cf571f2e70bf460349ad52744187e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:48,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:48,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:48,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332269069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:49,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332269079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:49,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:49,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:49,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d5886b21ae674c77b9b287e954d94397 2024-11-11T13:36:49,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/432d491b488640bb9fc61d87132ef739 is 50, key is test_row_0/B:col10/1731332208248/Put/seqid=0 2024-11-11T13:36:49,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742187_1363 (size=12151) 2024-11-11T13:36:49,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:49,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:49,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:49,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:49,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/432d491b488640bb9fc61d87132ef739 2024-11-11T13:36:49,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/4cc102a8782a44ffbe76fc96174ed426 is 50, key is test_row_0/C:col10/1731332208248/Put/seqid=0 2024-11-11T13:36:49,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332269580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:49,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742188_1364 (size=12151) 2024-11-11T13:36:49,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332269599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T13:36:49,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:49,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:49,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:49,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:49,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/4cc102a8782a44ffbe76fc96174ed426 2024-11-11T13:36:50,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d5886b21ae674c77b9b287e954d94397 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5886b21ae674c77b9b287e954d94397 2024-11-11T13:36:50,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5886b21ae674c77b9b287e954d94397, entries=200, sequenceid=167, filesize=38.8 K 2024-11-11T13:36:50,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/432d491b488640bb9fc61d87132ef739 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/432d491b488640bb9fc61d87132ef739 2024-11-11T13:36:50,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/432d491b488640bb9fc61d87132ef739, entries=150, sequenceid=167, filesize=11.9 K 2024-11-11T13:36:50,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/4cc102a8782a44ffbe76fc96174ed426 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/4cc102a8782a44ffbe76fc96174ed426 2024-11-11T13:36:50,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/4cc102a8782a44ffbe76fc96174ed426, entries=150, sequenceid=167, filesize=11.9 K 2024-11-11T13:36:50,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0e1cf571f2e70bf460349ad52744187e in 1776ms, sequenceid=167, compaction requested=false 2024-11-11T13:36:50,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:50,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T13:36:50,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:50,026 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:36:50,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111789d207bbaae48aebd61f7d403aae078_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332208413/Put/seqid=0 2024-11-11T13:36:50,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742189_1365 (size=12304) 2024-11-11T13:36:50,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,073 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111789d207bbaae48aebd61f7d403aae078_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111789d207bbaae48aebd61f7d403aae078_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/44ce8ddeb4f749d4a939969291e00d62, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:50,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/44ce8ddeb4f749d4a939969291e00d62 is 175, key is test_row_0/A:col10/1731332208413/Put/seqid=0 2024-11-11T13:36:50,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742190_1366 (size=31105) 2024-11-11T13:36:50,098 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/44ce8ddeb4f749d4a939969291e00d62 2024-11-11T13:36:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/8ee0b0bd09ee4faaafb299935abd5d93 is 50, key is test_row_0/B:col10/1731332208413/Put/seqid=0 2024-11-11T13:36:50,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742191_1367 (size=12151) 2024-11-11T13:36:50,127 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/8ee0b0bd09ee4faaafb299935abd5d93 2024-11-11T13:36:50,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/04eee5e917354baea72b3c933f0e98ac is 50, key is test_row_0/C:col10/1731332208413/Put/seqid=0 2024-11-11T13:36:50,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742192_1368 (size=12151) 2024-11-11T13:36:50,565 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/04eee5e917354baea72b3c933f0e98ac 2024-11-11T13:36:50,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/44ce8ddeb4f749d4a939969291e00d62 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/44ce8ddeb4f749d4a939969291e00d62 2024-11-11T13:36:50,577 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/44ce8ddeb4f749d4a939969291e00d62, entries=150, sequenceid=195, filesize=30.4 K 2024-11-11T13:36:50,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/8ee0b0bd09ee4faaafb299935abd5d93 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8ee0b0bd09ee4faaafb299935abd5d93 2024-11-11T13:36:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,583 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8ee0b0bd09ee4faaafb299935abd5d93, entries=150, sequenceid=195, filesize=11.9 K 2024-11-11T13:36:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/04eee5e917354baea72b3c933f0e98ac as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/04eee5e917354baea72b3c933f0e98ac 2024-11-11T13:36:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,587 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/04eee5e917354baea72b3c933f0e98ac, entries=150, sequenceid=195, filesize=11.9 K 2024-11-11T13:36:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,588 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 0e1cf571f2e70bf460349ad52744187e in 562ms, sequenceid=195, compaction requested=true 2024-11-11T13:36:50,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:50,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-11T13:36:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-11T13:36:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-11T13:36:50,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0660 sec 2024-11-11T13:36:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.0730 sec 2024-11-11T13:36:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,805 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:50,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:36:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119c6c361e528d4fc6933d392f8653d78e_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332210783/Put/seqid=0 2024-11-11T13:36:50,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742193_1369 (size=22268) 2024-11-11T13:36:50,895 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:50,909 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119c6c361e528d4fc6933d392f8653d78e_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119c6c361e528d4fc6933d392f8653d78e_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:50,920 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/b99ce59e18144826b542ce72054aa8f9, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:50,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/b99ce59e18144826b542ce72054aa8f9 is 175, key is test_row_0/A:col10/1731332210783/Put/seqid=0 2024-11-11T13:36:50,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742194_1370 (size=65673) 2024-11-11T13:36:50,959 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/b99ce59e18144826b542ce72054aa8f9 2024-11-11T13:36:50,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332270968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:50,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:50,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332270968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:50,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/1f56679b6cc945348b9b14e70d181b4a is 50, key is test_row_0/B:col10/1731332210783/Put/seqid=0 2024-11-11T13:36:50,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332270971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:50,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332270971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:50,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332270972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742195_1371 (size=12151) 2024-11-11T13:36:51,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/1f56679b6cc945348b9b14e70d181b4a 2024-11-11T13:36:51,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/03e4a838d2b746119a63934aa42d8ea3 is 50, key is test_row_0/C:col10/1731332210783/Put/seqid=0 2024-11-11T13:36:51,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742196_1372 (size=12151) 2024-11-11T13:36:51,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/03e4a838d2b746119a63934aa42d8ea3 2024-11-11T13:36:51,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/b99ce59e18144826b542ce72054aa8f9 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/b99ce59e18144826b542ce72054aa8f9 2024-11-11T13:36:51,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/b99ce59e18144826b542ce72054aa8f9, entries=350, sequenceid=206, filesize=64.1 K 2024-11-11T13:36:51,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/1f56679b6cc945348b9b14e70d181b4a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/1f56679b6cc945348b9b14e70d181b4a 2024-11-11T13:36:51,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/1f56679b6cc945348b9b14e70d181b4a, entries=150, sequenceid=206, filesize=11.9 K 2024-11-11T13:36:51,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/03e4a838d2b746119a63934aa42d8ea3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/03e4a838d2b746119a63934aa42d8ea3 2024-11-11T13:36:51,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/03e4a838d2b746119a63934aa42d8ea3, entries=150, sequenceid=206, filesize=11.9 K 2024-11-11T13:36:51,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 0e1cf571f2e70bf460349ad52744187e in 274ms, sequenceid=206, compaction requested=true 2024-11-11T13:36:51,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:51,079 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:51,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:51,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:51,081 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:51,090 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 167940 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:51,090 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/A is initiating minor compaction (all files) 2024-11-11T13:36:51,090 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/A in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:51,091 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d03a0ac63d254c5f92c175debe48b13e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5886b21ae674c77b9b287e954d94397, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/44ce8ddeb4f749d4a939969291e00d62, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/b99ce59e18144826b542ce72054aa8f9] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=164.0 K 2024-11-11T13:36:51,091 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:51,091 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d03a0ac63d254c5f92c175debe48b13e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5886b21ae674c77b9b287e954d94397, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/44ce8ddeb4f749d4a939969291e00d62, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/b99ce59e18144826b542ce72054aa8f9] 2024-11-11T13:36:51,091 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d03a0ac63d254c5f92c175debe48b13e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332206045 2024-11-11T13:36:51,092 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5886b21ae674c77b9b287e954d94397, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731332206693 2024-11-11T13:36:51,092 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44ce8ddeb4f749d4a939969291e00d62, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731332208361 2024-11-11T13:36:51,092 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting b99ce59e18144826b542ce72054aa8f9, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731332210678 2024-11-11T13:36:51,094 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:51,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-11T13:36:51,094 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:51,094 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/B is initiating minor compaction (all files) 2024-11-11T13:36:51,094 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/B in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:51,094 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/f91ad12b75b64cd8bb042bb3fa480f08, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/432d491b488640bb9fc61d87132ef739, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8ee0b0bd09ee4faaafb299935abd5d93, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/1f56679b6cc945348b9b14e70d181b4a] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=47.8 K 2024-11-11T13:36:51,095 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f91ad12b75b64cd8bb042bb3fa480f08, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332206045 2024-11-11T13:36:51,095 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 432d491b488640bb9fc61d87132ef739, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731332206709 2024-11-11T13:36:51,095 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ee0b0bd09ee4faaafb299935abd5d93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731332208361 2024-11-11T13:36:51,096 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f56679b6cc945348b9b14e70d181b4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731332210730 2024-11-11T13:36:51,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332271104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332271104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332271116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,127 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:51,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332271118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332271119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,144 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#B#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:51,144 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/1dd7f59ac81844a383cc01d472e623d5 is 50, key is test_row_0/B:col10/1731332210783/Put/seqid=0 2024-11-11T13:36:51,145 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111109654cb9d04c42888e5e56f909536f00_0e1cf571f2e70bf460349ad52744187e store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:51,147 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111109654cb9d04c42888e5e56f909536f00_0e1cf571f2e70bf460349ad52744187e, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:51,148 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111109654cb9d04c42888e5e56f909536f00_0e1cf571f2e70bf460349ad52744187e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:51,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f9e7f7c260ad4a5390e0115e13ea72a8_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332210970/Put/seqid=0 2024-11-11T13:36:51,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742197_1373 (size=12595) 2024-11-11T13:36:51,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742199_1375 (size=19774) 2024-11-11T13:36:51,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332271230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332271230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742198_1374 (size=4469) 2024-11-11T13:36:51,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332271240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332271240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332271332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332271450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332271450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332271461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332271461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,598 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/1dd7f59ac81844a383cc01d472e623d5 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/1dd7f59ac81844a383cc01d472e623d5 2024-11-11T13:36:51,604 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/B of 0e1cf571f2e70bf460349ad52744187e into 1dd7f59ac81844a383cc01d472e623d5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:51,604 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:51,604 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/B, priority=12, startTime=1731332211081; duration=0sec 2024-11-11T13:36:51,604 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:51,604 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:B 2024-11-11T13:36:51,604 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:36:51,605 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:36:51,605 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/C is initiating minor compaction (all files) 2024-11-11T13:36:51,605 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/C in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:51,606 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/4498b617b2784352a03a7645df1027cb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/4cc102a8782a44ffbe76fc96174ed426, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/04eee5e917354baea72b3c933f0e98ac, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/03e4a838d2b746119a63934aa42d8ea3] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=47.8 K 2024-11-11T13:36:51,606 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 4498b617b2784352a03a7645df1027cb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332206045 2024-11-11T13:36:51,606 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cc102a8782a44ffbe76fc96174ed426, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731332206709 2024-11-11T13:36:51,606 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 04eee5e917354baea72b3c933f0e98ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731332208361 2024-11-11T13:36:51,607 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 03e4a838d2b746119a63934aa42d8ea3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731332210730 2024-11-11T13:36:51,614 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:51,616 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/c6a5f099101a4119bddbaff00a6c46eb is 50, key is test_row_0/C:col10/1731332210783/Put/seqid=0 2024-11-11T13:36:51,619 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:51,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742200_1376 (size=12595) 2024-11-11T13:36:51,625 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f9e7f7c260ad4a5390e0115e13ea72a8_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f9e7f7c260ad4a5390e0115e13ea72a8_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:51,626 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/a3d9862912514f2f85e7210d15f26753, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:51,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/a3d9862912514f2f85e7210d15f26753 is 175, key is test_row_0/A:col10/1731332210970/Put/seqid=0 2024-11-11T13:36:51,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742201_1377 (size=57033) 2024-11-11T13:36:51,646 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#A#compaction#315 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:51,646 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/14b72c1989d54504b6ccff3348901083 is 175, key is test_row_0/A:col10/1731332210783/Put/seqid=0 2024-11-11T13:36:51,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742202_1378 (size=31549) 2024-11-11T13:36:51,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332271649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T13:36:51,690 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:51,690 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-11T13:36:51,692 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T13:36:51,693 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:51,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:51,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332271759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332271769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332271779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332271780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T13:36:51,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:51,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:51,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:51,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:51,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:52,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:52,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T13:36:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,041 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/c6a5f099101a4119bddbaff00a6c46eb as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/c6a5f099101a4119bddbaff00a6c46eb 2024-11-11T13:36:52,044 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/a3d9862912514f2f85e7210d15f26753 2024-11-11T13:36:52,049 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/C of 0e1cf571f2e70bf460349ad52744187e into c6a5f099101a4119bddbaff00a6c46eb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:52,049 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:52,049 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/C, priority=12, startTime=1731332211081; duration=0sec 2024-11-11T13:36:52,049 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:52,049 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:C 2024-11-11T13:36:52,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/2c1cd694dcd743b8980a44141aba3335 is 50, key is test_row_0/B:col10/1731332210970/Put/seqid=0 2024-11-11T13:36:52,064 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/14b72c1989d54504b6ccff3348901083 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/14b72c1989d54504b6ccff3348901083 2024-11-11T13:36:52,071 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/A of 0e1cf571f2e70bf460349ad52744187e into 14b72c1989d54504b6ccff3348901083(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:52,071 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:52,071 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/A, priority=12, startTime=1731332211079; duration=0sec 2024-11-11T13:36:52,071 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:52,071 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:A 2024-11-11T13:36:52,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742203_1379 (size=12151) 2024-11-11T13:36:52,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/2c1cd694dcd743b8980a44141aba3335 2024-11-11T13:36:52,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/e4a75ebe523c4911a684a8ee599585a2 is 50, key is test_row_0/C:col10/1731332210970/Put/seqid=0 2024-11-11T13:36:52,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742204_1380 (size=12151) 2024-11-11T13:36:52,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/e4a75ebe523c4911a684a8ee599585a2 2024-11-11T13:36:52,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/a3d9862912514f2f85e7210d15f26753 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/a3d9862912514f2f85e7210d15f26753 2024-11-11T13:36:52,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332272161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:52,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/a3d9862912514f2f85e7210d15f26753, entries=300, sequenceid=235, filesize=55.7 K 2024-11-11T13:36:52,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/2c1cd694dcd743b8980a44141aba3335 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c1cd694dcd743b8980a44141aba3335 2024-11-11T13:36:52,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c1cd694dcd743b8980a44141aba3335, entries=150, sequenceid=235, filesize=11.9 K 2024-11-11T13:36:52,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/e4a75ebe523c4911a684a8ee599585a2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e4a75ebe523c4911a684a8ee599585a2 2024-11-11T13:36:52,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e4a75ebe523c4911a684a8ee599585a2, entries=150, sequenceid=235, filesize=11.9 K 2024-11-11T13:36:52,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 0e1cf571f2e70bf460349ad52744187e in 1107ms, sequenceid=235, compaction requested=false 2024-11-11T13:36:52,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:52,282 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:52,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:36:52,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113b9b1dd9c0aa4f3db9d34b46b2d2524c_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332211117/Put/seqid=0 2024-11-11T13:36:52,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742205_1381 (size=12304) 2024-11-11T13:36:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T13:36:52,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332272339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332272342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332272342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332272349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332272460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332272460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332272460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332272463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:52,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332272669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332272669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332272669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332272679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,696 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:52,705 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113b9b1dd9c0aa4f3db9d34b46b2d2524c_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113b9b1dd9c0aa4f3db9d34b46b2d2524c_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:52,706 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/5ea0f608ce334e38aeb889cf19fd65fd, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:52,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/5ea0f608ce334e38aeb889cf19fd65fd is 175, key is test_row_0/A:col10/1731332211117/Put/seqid=0 2024-11-11T13:36:52,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742206_1382 (size=31105) 2024-11-11T13:36:52,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:52,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:52,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T13:36:52,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:52,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:52,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:52,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332272980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332272980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332272980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:53,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:53,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332272992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:53,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:53,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:53,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,140 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/5ea0f608ce334e38aeb889cf19fd65fd 2024-11-11T13:36:53,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/9dca03836206461ab2fa9bfbf7b76d18 is 50, key is test_row_0/B:col10/1731332211117/Put/seqid=0 2024-11-11T13:36:53,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742207_1383 (size=12151) 2024-11-11T13:36:53,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332273183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:53,189 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_2' in 0e1cf571f2e70bf460349ad52744187e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:36:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:53,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:53,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:53,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:53,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:53,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332273499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:53,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332273499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:53,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332273499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:53,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332273509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:53,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:53,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:53,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/9dca03836206461ab2fa9bfbf7b76d18 2024-11-11T13:36:53,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/6b74633f935e42d8a8b3ca82695bf382 is 50, key is test_row_0/C:col10/1731332211117/Put/seqid=0 2024-11-11T13:36:53,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:53,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:53,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742208_1384 (size=12151) 2024-11-11T13:36:53,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:53,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:53,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T13:36:53,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:53,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:53,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:53,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:53,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/6b74633f935e42d8a8b3ca82695bf382 2024-11-11T13:36:53,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/5ea0f608ce334e38aeb889cf19fd65fd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/5ea0f608ce334e38aeb889cf19fd65fd 2024-11-11T13:36:54,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/5ea0f608ce334e38aeb889cf19fd65fd, entries=150, sequenceid=249, filesize=30.4 K 2024-11-11T13:36:54,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/9dca03836206461ab2fa9bfbf7b76d18 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/9dca03836206461ab2fa9bfbf7b76d18 2024-11-11T13:36:54,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/9dca03836206461ab2fa9bfbf7b76d18, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T13:36:54,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/6b74633f935e42d8a8b3ca82695bf382 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/6b74633f935e42d8a8b3ca82695bf382 2024-11-11T13:36:54,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/6b74633f935e42d8a8b3ca82695bf382, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T13:36:54,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0e1cf571f2e70bf460349ad52744187e in 1739ms, sequenceid=249, compaction requested=true 2024-11-11T13:36:54,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:54,021 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:54,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:54,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:54,022 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:54,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:54,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:54,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:54,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:54,027 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:54,027 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:54,027 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/A is initiating minor compaction (all files) 2024-11-11T13:36:54,027 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/B is initiating minor compaction (all files) 2024-11-11T13:36:54,027 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/A in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:54,027 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/B in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:54,028 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/1dd7f59ac81844a383cc01d472e623d5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c1cd694dcd743b8980a44141aba3335, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/9dca03836206461ab2fa9bfbf7b76d18] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=36.0 K 2024-11-11T13:36:54,028 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/14b72c1989d54504b6ccff3348901083, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/a3d9862912514f2f85e7210d15f26753, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/5ea0f608ce334e38aeb889cf19fd65fd] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=116.9 K 2024-11-11T13:36:54,028 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:54,028 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/14b72c1989d54504b6ccff3348901083, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/a3d9862912514f2f85e7210d15f26753, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/5ea0f608ce334e38aeb889cf19fd65fd] 2024-11-11T13:36:54,028 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14b72c1989d54504b6ccff3348901083, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731332210730 2024-11-11T13:36:54,028 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dd7f59ac81844a383cc01d472e623d5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731332210730 2024-11-11T13:36:54,028 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c1cd694dcd743b8980a44141aba3335, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731332210970 2024-11-11T13:36:54,028 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d9862912514f2f85e7210d15f26753, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731332210948 2024-11-11T13:36:54,029 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dca03836206461ab2fa9bfbf7b76d18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731332211114 2024-11-11T13:36:54,029 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ea0f608ce334e38aeb889cf19fd65fd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731332211114 2024-11-11T13:36:54,035 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:54,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T13:36:54,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:54,044 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:36:54,049 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#B#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:54,050 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/69b32a34f4b64d30b1ba565da883d765 is 50, key is test_row_0/B:col10/1731332211117/Put/seqid=0 2024-11-11T13:36:54,060 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111e6e1c2a3286a4431bd6a659e6e064f13_0e1cf571f2e70bf460349ad52744187e store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:54,062 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111e6e1c2a3286a4431bd6a659e6e064f13_0e1cf571f2e70bf460349ad52744187e, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:54,062 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e6e1c2a3286a4431bd6a659e6e064f13_0e1cf571f2e70bf460349ad52744187e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:54,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111142d8e8a9b3f04ab3a27a9ec5cb4be714_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332212340/Put/seqid=0 2024-11-11T13:36:54,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742210_1386 (size=4469) 2024-11-11T13:36:54,098 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#A#compaction#324 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:54,099 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d67d5aa91f094e2a9c274037be98de40 is 175, key is test_row_0/A:col10/1731332211117/Put/seqid=0 2024-11-11T13:36:54,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742209_1385 (size=12697) 2024-11-11T13:36:54,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742212_1388 (size=31651) 2024-11-11T13:36:54,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742211_1387 (size=12454) 2024-11-11T13:36:54,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:54,536 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/69b32a34f4b64d30b1ba565da883d765 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/69b32a34f4b64d30b1ba565da883d765 2024-11-11T13:36:54,548 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/B of 0e1cf571f2e70bf460349ad52744187e into 69b32a34f4b64d30b1ba565da883d765(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:54,548 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:54,548 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/B, priority=13, startTime=1731332214022; duration=0sec 2024-11-11T13:36:54,548 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:54,548 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:B 2024-11-11T13:36:54,548 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:54,550 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:54,550 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/C is initiating minor compaction (all files) 2024-11-11T13:36:54,550 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/C in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:54,550 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/c6a5f099101a4119bddbaff00a6c46eb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e4a75ebe523c4911a684a8ee599585a2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/6b74633f935e42d8a8b3ca82695bf382] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=36.0 K 2024-11-11T13:36:54,551 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a5f099101a4119bddbaff00a6c46eb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731332210730 2024-11-11T13:36:54,552 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e4a75ebe523c4911a684a8ee599585a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731332210970 2024-11-11T13:36:54,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332274544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332274546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,553 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b74633f935e42d8a8b3ca82695bf382, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731332211114 2024-11-11T13:36:54,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:54,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332274549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:54,562 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/d67d5aa91f094e2a9c274037be98de40 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d67d5aa91f094e2a9c274037be98de40 2024-11-11T13:36:54,567 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/A of 0e1cf571f2e70bf460349ad52744187e into d67d5aa91f094e2a9c274037be98de40(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:54,567 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:54,567 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/A, priority=13, startTime=1731332214021; duration=0sec 2024-11-11T13:36:54,567 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:54,567 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:A 2024-11-11T13:36:54,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332274551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,570 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:54,570 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/9207c0a88475406ba4a45490daf17b6e is 50, key is test_row_0/C:col10/1731332211117/Put/seqid=0 2024-11-11T13:36:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:54,609 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111142d8e8a9b3f04ab3a27a9ec5cb4be714_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111142d8e8a9b3f04ab3a27a9ec5cb4be714_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:54,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/2b0f142d00384e64b08fafaf1c24f2e0, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:54,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/2b0f142d00384e64b08fafaf1c24f2e0 is 175, key is test_row_0/A:col10/1731332212340/Put/seqid=0 2024-11-11T13:36:54,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742213_1389 (size=12697) 2024-11-11T13:36:54,660 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/9207c0a88475406ba4a45490daf17b6e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/9207c0a88475406ba4a45490daf17b6e 2024-11-11T13:36:54,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742214_1390 (size=31255) 2024-11-11T13:36:54,663 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/2b0f142d00384e64b08fafaf1c24f2e0 2024-11-11T13:36:54,666 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/C of 0e1cf571f2e70bf460349ad52744187e into 9207c0a88475406ba4a45490daf17b6e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:54,666 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:54,666 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/C, priority=13, startTime=1731332214022; duration=0sec 2024-11-11T13:36:54,666 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:54,666 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:C 2024-11-11T13:36:54,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/2c14e6a00dc24057a65f8a815b9eb779 is 50, key is test_row_0/B:col10/1731332212340/Put/seqid=0 2024-11-11T13:36:54,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332274660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:54,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332274660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:54,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742215_1391 (size=12301) 2024-11-11T13:36:54,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332274669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:54,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332274679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:54,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332274879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:54,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332274879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:54,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332274889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:54,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332274899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:54,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,087 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/2c14e6a00dc24057a65f8a815b9eb779 2024-11-11T13:36:55,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/93baa6a5bd55490785d7ec302404f5a4 is 50, key is test_row_0/C:col10/1731332212340/Put/seqid=0 2024-11-11T13:36:55,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742216_1392 (size=12301) 2024-11-11T13:36:55,108 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/93baa6a5bd55490785d7ec302404f5a4 2024-11-11T13:36:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/2b0f142d00384e64b08fafaf1c24f2e0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2b0f142d00384e64b08fafaf1c24f2e0 2024-11-11T13:36:55,116 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2b0f142d00384e64b08fafaf1c24f2e0, entries=150, sequenceid=274, filesize=30.5 K 2024-11-11T13:36:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/2c14e6a00dc24057a65f8a815b9eb779 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c14e6a00dc24057a65f8a815b9eb779 2024-11-11T13:36:55,121 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c14e6a00dc24057a65f8a815b9eb779, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T13:36:55,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/93baa6a5bd55490785d7ec302404f5a4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/93baa6a5bd55490785d7ec302404f5a4 2024-11-11T13:36:55,127 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/93baa6a5bd55490785d7ec302404f5a4, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T13:36:55,128 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0e1cf571f2e70bf460349ad52744187e in 1084ms, sequenceid=274, compaction requested=false 2024-11-11T13:36:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-11T13:36:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-11T13:36:55,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T13:36:55,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4360 sec 2024-11-11T13:36:55,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.4410 sec 2024-11-11T13:36:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:55,194 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:55,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T13:36:55,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111166ff4256092341a29776aba2c09d7939_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332214545/Put/seqid=0 2024-11-11T13:36:55,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742217_1393 (size=17534) 2024-11-11T13:36:55,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332275281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332275282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332275283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332275283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332275284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332275390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332275399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332275399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332275399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332275400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,607 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:55,615 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111166ff4256092341a29776aba2c09d7939_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111166ff4256092341a29776aba2c09d7939_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:55,618 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/13dcb0a552db4358a662f88a6c7376d7, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:55,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/13dcb0a552db4358a662f88a6c7376d7 is 175, key is test_row_0/A:col10/1731332214545/Put/seqid=0 2024-11-11T13:36:55,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332275609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332275621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332275621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332275623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332275623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742218_1394 (size=48639) 2024-11-11T13:36:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T13:36:55,849 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:55,850 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-11T13:36:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T13:36:55,851 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:55,852 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:55,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:55,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332275929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332275939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332275939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332275949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332275949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:55,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T13:36:56,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T13:36:56,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:56,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,061 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/13dcb0a552db4358a662f88a6c7376d7 2024-11-11T13:36:56,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/84ea751b10884ae1a2766645d752fa57 is 50, key is test_row_0/B:col10/1731332214545/Put/seqid=0 2024-11-11T13:36:56,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742219_1395 (size=12301) 2024-11-11T13:36:56,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T13:36:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:56,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T13:36:56,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T13:36:56,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:56,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332276450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:56,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332276450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:56,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332276452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:56,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T13:36:56,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:56,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:56,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:56,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:56,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332276467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332276469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:56,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/84ea751b10884ae1a2766645d752fa57 2024-11-11T13:36:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T13:36:56,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/2953daebac79472a98a9921c1ca27fac is 50, key is test_row_0/C:col10/1731332214545/Put/seqid=0 2024-11-11T13:36:56,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742220_1396 (size=12301) 2024-11-11T13:36:56,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/2953daebac79472a98a9921c1ca27fac 2024-11-11T13:36:56,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/13dcb0a552db4358a662f88a6c7376d7 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/13dcb0a552db4358a662f88a6c7376d7 2024-11-11T13:36:56,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/13dcb0a552db4358a662f88a6c7376d7, entries=250, sequenceid=290, filesize=47.5 K 2024-11-11T13:36:56,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/84ea751b10884ae1a2766645d752fa57 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/84ea751b10884ae1a2766645d752fa57 2024-11-11T13:36:56,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/84ea751b10884ae1a2766645d752fa57, entries=150, sequenceid=290, filesize=12.0 K 2024-11-11T13:36:56,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/2953daebac79472a98a9921c1ca27fac as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/2953daebac79472a98a9921c1ca27fac 2024-11-11T13:36:56,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/2953daebac79472a98a9921c1ca27fac, entries=150, sequenceid=290, filesize=12.0 K 2024-11-11T13:36:56,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 0e1cf571f2e70bf460349ad52744187e in 1313ms, sequenceid=290, compaction requested=true 2024-11-11T13:36:56,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:56,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:56,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:56,507 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:56,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:56,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:56,507 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:56,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:56,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:56,508 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111545 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:56,508 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:56,508 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/B is initiating minor compaction (all files) 2024-11-11T13:36:56,508 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/A is initiating minor compaction (all files) 2024-11-11T13:36:56,508 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/B in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,508 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/A in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,508 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/69b32a34f4b64d30b1ba565da883d765, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c14e6a00dc24057a65f8a815b9eb779, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/84ea751b10884ae1a2766645d752fa57] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=36.4 K 2024-11-11T13:36:56,508 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d67d5aa91f094e2a9c274037be98de40, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2b0f142d00384e64b08fafaf1c24f2e0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/13dcb0a552db4358a662f88a6c7376d7] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=108.9 K 2024-11-11T13:36:56,508 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,508 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d67d5aa91f094e2a9c274037be98de40, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2b0f142d00384e64b08fafaf1c24f2e0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/13dcb0a552db4358a662f88a6c7376d7] 2024-11-11T13:36:56,508 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b32a34f4b64d30b1ba565da883d765, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731332211114 2024-11-11T13:36:56,509 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d67d5aa91f094e2a9c274037be98de40, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731332211114 2024-11-11T13:36:56,509 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c14e6a00dc24057a65f8a815b9eb779, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731332212321 2024-11-11T13:36:56,509 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b0f142d00384e64b08fafaf1c24f2e0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731332212321 2024-11-11T13:36:56,509 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 84ea751b10884ae1a2766645d752fa57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731332214545 2024-11-11T13:36:56,509 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13dcb0a552db4358a662f88a6c7376d7, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731332214519 2024-11-11T13:36:56,515 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:56,521 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:56,522 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/eb3b569dcc514a18bcae6320e9bfc703 is 50, key is test_row_0/B:col10/1731332214545/Put/seqid=0 2024-11-11T13:36:56,524 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111a4ae31bebd1e4e6ab1ed9aed265df20d_0e1cf571f2e70bf460349ad52744187e store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:56,527 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111a4ae31bebd1e4e6ab1ed9aed265df20d_0e1cf571f2e70bf460349ad52744187e, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:56,527 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a4ae31bebd1e4e6ab1ed9aed265df20d_0e1cf571f2e70bf460349ad52744187e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:56,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742221_1397 (size=12949) 2024-11-11T13:36:56,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742222_1398 (size=4469) 2024-11-11T13:36:56,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T13:36:56,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,619 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T13:36:56,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111e7f24d37b374c9b86f94c284730fc20_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332215282/Put/seqid=0 2024-11-11T13:36:56,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742223_1399 (size=12454) 2024-11-11T13:36:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:56,660 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111e7f24d37b374c9b86f94c284730fc20_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111e7f24d37b374c9b86f94c284730fc20_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/da7b7ef9aac741299e030c53a4a4fc76, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/da7b7ef9aac741299e030c53a4a4fc76 is 175, key is test_row_0/A:col10/1731332215282/Put/seqid=0 2024-11-11T13:36:56,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742224_1400 (size=31255) 2024-11-11T13:36:56,949 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/eb3b569dcc514a18bcae6320e9bfc703 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/eb3b569dcc514a18bcae6320e9bfc703 2024-11-11T13:36:56,953 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/B of 0e1cf571f2e70bf460349ad52744187e into eb3b569dcc514a18bcae6320e9bfc703(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:56,953 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:56,953 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/B, priority=13, startTime=1731332216507; duration=0sec 2024-11-11T13:36:56,953 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:56,953 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:B 2024-11-11T13:36:56,953 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:56,953 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:56,953 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/C is initiating minor compaction (all files) 2024-11-11T13:36:56,953 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/C in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:56,954 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/9207c0a88475406ba4a45490daf17b6e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/93baa6a5bd55490785d7ec302404f5a4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/2953daebac79472a98a9921c1ca27fac] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=36.4 K 2024-11-11T13:36:56,954 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 9207c0a88475406ba4a45490daf17b6e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731332211114 2024-11-11T13:36:56,954 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 93baa6a5bd55490785d7ec302404f5a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731332212321 2024-11-11T13:36:56,954 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2953daebac79472a98a9921c1ca27fac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731332214545 2024-11-11T13:36:56,961 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:56,961 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/f4267674bc2b45a583da5a6bb11c6dfd is 50, key is test_row_0/C:col10/1731332214545/Put/seqid=0 2024-11-11T13:36:56,965 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#A#compaction#333 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:56,966 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/0357276afe994da780b97dc442a61a8d is 175, key is test_row_0/A:col10/1731332214545/Put/seqid=0 2024-11-11T13:36:56,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742225_1401 (size=12949) 2024-11-11T13:36:56,980 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/f4267674bc2b45a583da5a6bb11c6dfd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/f4267674bc2b45a583da5a6bb11c6dfd 2024-11-11T13:36:56,986 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/C of 0e1cf571f2e70bf460349ad52744187e into f4267674bc2b45a583da5a6bb11c6dfd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:56,986 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:56,986 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/C, priority=13, startTime=1731332216507; duration=0sec 2024-11-11T13:36:56,986 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:56,986 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:C 2024-11-11T13:36:56,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742226_1402 (size=31903) 2024-11-11T13:36:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T13:36:56,995 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/0357276afe994da780b97dc442a61a8d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/0357276afe994da780b97dc442a61a8d 2024-11-11T13:36:56,999 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/A of 0e1cf571f2e70bf460349ad52744187e into 0357276afe994da780b97dc442a61a8d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:56,999 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:56,999 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/A, priority=13, startTime=1731332216507; duration=0sec 2024-11-11T13:36:56,999 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:56,999 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:A 2024-11-11T13:36:57,065 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/da7b7ef9aac741299e030c53a4a4fc76 2024-11-11T13:36:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/78cd6a3b250c4650aba46ab53a95f681 is 50, key is test_row_0/B:col10/1731332215282/Put/seqid=0 2024-11-11T13:36:57,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742227_1403 (size=12301) 2024-11-11T13:36:57,093 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/78cd6a3b250c4650aba46ab53a95f681 2024-11-11T13:36:57,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/f3e3be9fb50d43949c6b3d6d436a3aed is 50, key is test_row_0/C:col10/1731332215282/Put/seqid=0 2024-11-11T13:36:57,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742228_1404 (size=12301) 2024-11-11T13:36:57,108 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/f3e3be9fb50d43949c6b3d6d436a3aed 2024-11-11T13:36:57,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/da7b7ef9aac741299e030c53a4a4fc76 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/da7b7ef9aac741299e030c53a4a4fc76 2024-11-11T13:36:57,116 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/da7b7ef9aac741299e030c53a4a4fc76, entries=150, sequenceid=314, filesize=30.5 K 2024-11-11T13:36:57,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/78cd6a3b250c4650aba46ab53a95f681 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/78cd6a3b250c4650aba46ab53a95f681 2024-11-11T13:36:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,120 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/78cd6a3b250c4650aba46ab53a95f681, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T13:36:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/f3e3be9fb50d43949c6b3d6d436a3aed as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/f3e3be9fb50d43949c6b3d6d436a3aed 2024-11-11T13:36:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,124 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/f3e3be9fb50d43949c6b3d6d436a3aed, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T13:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,124 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 0e1cf571f2e70bf460349ad52744187e in 505ms, sequenceid=314, compaction requested=false 2024-11-11T13:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:57,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-11T13:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-11T13:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T13:36:57,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2740 sec 2024-11-11T13:36:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.2780 sec 2024-11-11T13:36:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:57,614 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:36:57,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:36:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:57,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111611427ccf3b94726a67e612bd5ad7fab_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332217612/Put/seqid=0 2024-11-11T13:36:57,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742230_1406 (size=25158) 2024-11-11T13:36:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332277663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:57,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332277679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:57,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332277680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:57,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332277681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332277681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:57,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332277790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332277799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332277799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:57,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332277799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:57,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332277799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:57,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T13:36:58,007 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:36:58,008 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:36:58,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-11T13:36:58,011 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:36:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T13:36:58,012 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:36:58,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:36:58,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332278010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332278020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332278023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332278023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332278023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,078 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:58,081 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111611427ccf3b94726a67e612bd5ad7fab_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111611427ccf3b94726a67e612bd5ad7fab_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:58,081 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/2558ca9ada60401c8895a64f49b783ef, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:58,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/2558ca9ada60401c8895a64f49b783ef is 175, key is test_row_0/A:col10/1731332217612/Put/seqid=0 2024-11-11T13:36:58,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742229_1405 (size=74795) 2024-11-11T13:36:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T13:36:58,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T13:36:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T13:36:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T13:36:58,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332278329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332278329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332278350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332278350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332278350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T13:36:58,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:58,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,484 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/2558ca9ada60401c8895a64f49b783ef 2024-11-11T13:36:58,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/36a670abcb244ac48db41d0d59c2f626 is 50, key is test_row_0/B:col10/1731332217612/Put/seqid=0 2024-11-11T13:36:58,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742231_1407 (size=12301) 2024-11-11T13:36:58,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T13:36:58,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:58,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T13:36:58,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T13:36:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43640 deadline: 1731332278839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43590 deadline: 1731332278839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43586 deadline: 1731332278860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43614 deadline: 1731332278860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:36:58,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43656 deadline: 1731332278860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:36:58,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:36:58,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:36:58,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/36a670abcb244ac48db41d0d59c2f626 2024-11-11T13:36:58,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/1423b92008174b68b5ff2a4080e5a4cd is 50, key is test_row_0/C:col10/1731332217612/Put/seqid=0 2024-11-11T13:36:58,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742232_1408 (size=12301) 2024-11-11T13:36:58,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/1423b92008174b68b5ff2a4080e5a4cd 2024-11-11T13:36:58,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/2558ca9ada60401c8895a64f49b783ef as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2558ca9ada60401c8895a64f49b783ef 2024-11-11T13:36:58,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T13:36:58,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:58,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:58,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:36:58,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2558ca9ada60401c8895a64f49b783ef, entries=400, sequenceid=328, filesize=73.0 K 2024-11-11T13:36:58,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/36a670abcb244ac48db41d0d59c2f626 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/36a670abcb244ac48db41d0d59c2f626 2024-11-11T13:36:58,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/36a670abcb244ac48db41d0d59c2f626, entries=150, sequenceid=328, filesize=12.0 K 2024-11-11T13:36:58,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/1423b92008174b68b5ff2a4080e5a4cd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/1423b92008174b68b5ff2a4080e5a4cd 2024-11-11T13:36:59,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/1423b92008174b68b5ff2a4080e5a4cd, entries=150, sequenceid=328, filesize=12.0 K 2024-11-11T13:36:59,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0e1cf571f2e70bf460349ad52744187e in 1391ms, sequenceid=328, compaction requested=true 2024-11-11T13:36:59,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:59,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:36:59,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:59,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:36:59,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:36:59,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e1cf571f2e70bf460349ad52744187e:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:36:59,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T13:36:59,007 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:59,007 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:59,008 INFO [Thread-1490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1490. 2024-11-11T13:36:59,008 DEBUG [Thread-1490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:59,008 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,008 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:59,009 INFO [Thread-1488 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1488. 2024-11-11T13:36:59,010 DEBUG [Thread-1488 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:59,010 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,010 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,012 INFO [Thread-1484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1484. 2024-11-11T13:36:59,012 DEBUG [Thread-1484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:59,012 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,013 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:59,013 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/C is initiating minor compaction (all files) 2024-11-11T13:36:59,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:59,013 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/C in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:59,013 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,013 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/f4267674bc2b45a583da5a6bb11c6dfd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/f3e3be9fb50d43949c6b3d6d436a3aed, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/1423b92008174b68b5ff2a4080e5a4cd] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=36.7 K 2024-11-11T13:36:59,013 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f4267674bc2b45a583da5a6bb11c6dfd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731332214545 2024-11-11T13:36:59,013 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137953 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:59,013 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/A is initiating minor compaction (all files) 2024-11-11T13:36:59,014 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/A in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:59,014 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f3e3be9fb50d43949c6b3d6d436a3aed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731332215266 2024-11-11T13:36:59,014 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/0357276afe994da780b97dc442a61a8d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/da7b7ef9aac741299e030c53a4a4fc76, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2558ca9ada60401c8895a64f49b783ef] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=134.7 K 2024-11-11T13:36:59,014 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:59,014 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/0357276afe994da780b97dc442a61a8d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/da7b7ef9aac741299e030c53a4a4fc76, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2558ca9ada60401c8895a64f49b783ef] 2024-11-11T13:36:59,014 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1423b92008174b68b5ff2a4080e5a4cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731332217586 2024-11-11T13:36:59,014 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0357276afe994da780b97dc442a61a8d, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731332214545 2024-11-11T13:36:59,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:59,015 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7b7ef9aac741299e030c53a4a4fc76, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731332215266 2024-11-11T13:36:59,016 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2558ca9ada60401c8895a64f49b783ef, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731332217550 2024-11-11T13:36:59,019 INFO [Thread-1482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1482. 2024-11-11T13:36:59,019 DEBUG [Thread-1482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:59,019 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,020 DEBUG [Thread-1482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:59,022 INFO [Thread-1486 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1486. 2024-11-11T13:36:59,022 DEBUG [Thread-1486 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:59,022 DEBUG [Thread-1486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,023 DEBUG [Thread-1486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,024 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#C#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:59,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:59,024 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:59,024 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/c4bbf19011ed49bb89da2ddf78b2d5a5 is 50, key is test_row_0/C:col10/1731332217612/Put/seqid=0 2024-11-11T13:36:59,032 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411116f04c07e5a804391983892dabbd5d506_0e1cf571f2e70bf460349ad52744187e store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:59,035 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411116f04c07e5a804391983892dabbd5d506_0e1cf571f2e70bf460349ad52744187e, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:59,035 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116f04c07e5a804391983892dabbd5d506_0e1cf571f2e70bf460349ad52744187e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:59,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742234_1410 (size=4469) 2024-11-11T13:36:59,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742233_1409 (size=13051) 2024-11-11T13:36:59,056 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/c4bbf19011ed49bb89da2ddf78b2d5a5 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/c4bbf19011ed49bb89da2ddf78b2d5a5 2024-11-11T13:36:59,062 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/C of 0e1cf571f2e70bf460349ad52744187e into c4bbf19011ed49bb89da2ddf78b2d5a5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:59,062 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:59,062 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/C, priority=13, startTime=1731332219005; duration=0sec 2024-11-11T13:36:59,063 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:36:59,063 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:C 2024-11-11T13:36:59,063 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:36:59,064 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:36:59,064 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): 0e1cf571f2e70bf460349ad52744187e/B is initiating minor compaction (all files) 2024-11-11T13:36:59,064 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e1cf571f2e70bf460349ad52744187e/B in TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:59,064 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/eb3b569dcc514a18bcae6320e9bfc703, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/78cd6a3b250c4650aba46ab53a95f681, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/36a670abcb244ac48db41d0d59c2f626] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp, totalSize=36.7 K 2024-11-11T13:36:59,065 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting eb3b569dcc514a18bcae6320e9bfc703, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731332214545 2024-11-11T13:36:59,065 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 78cd6a3b250c4650aba46ab53a95f681, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731332215266 2024-11-11T13:36:59,065 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 36a670abcb244ac48db41d0d59c2f626, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731332217586 2024-11-11T13:36:59,070 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#B#compaction#344 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:59,071 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/7c7e5bd8eb934b07afb36a86fcf830d8 is 50, key is test_row_0/B:col10/1731332217612/Put/seqid=0 2024-11-11T13:36:59,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742235_1411 (size=13051) 2024-11-11T13:36:59,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T13:36:59,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:59,098 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:36:59,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110f8d2eb4f41e47a3a42ea8eaf2db5655_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332217678/Put/seqid=0 2024-11-11T13:36:59,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742236_1412 (size=12454) 2024-11-11T13:36:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T13:36:59,445 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e1cf571f2e70bf460349ad52744187e#A#compaction#343 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:36:59,446 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/fa31cfc730bd49448dcc72e64e9fa818 is 175, key is test_row_0/A:col10/1731332217612/Put/seqid=0 2024-11-11T13:36:59,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742237_1413 (size=32005) 2024-11-11T13:36:59,479 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/7c7e5bd8eb934b07afb36a86fcf830d8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/7c7e5bd8eb934b07afb36a86fcf830d8 2024-11-11T13:36:59,483 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/B of 0e1cf571f2e70bf460349ad52744187e into 7c7e5bd8eb934b07afb36a86fcf830d8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:59,483 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:59,483 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/B, priority=13, startTime=1731332219005; duration=0sec 2024-11-11T13:36:59,483 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:59,483 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:B 2024-11-11T13:36:59,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:36:59,513 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110f8d2eb4f41e47a3a42ea8eaf2db5655_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110f8d2eb4f41e47a3a42ea8eaf2db5655_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/1424b05ec86c4de5bfb58259b774474d, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:36:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/1424b05ec86c4de5bfb58259b774474d is 175, key is test_row_0/A:col10/1731332217678/Put/seqid=0 2024-11-11T13:36:59,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742238_1414 (size=31255) 2024-11-11T13:36:59,518 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/1424b05ec86c4de5bfb58259b774474d 2024-11-11T13:36:59,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/bc02117414504409b9ed14a924e455a6 is 50, key is test_row_0/B:col10/1731332217678/Put/seqid=0 2024-11-11T13:36:59,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742239_1415 (size=12301) 2024-11-11T13:36:59,532 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/bc02117414504409b9ed14a924e455a6 2024-11-11T13:36:59,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/e0a3664f7bce4d1eae1efbf49ae837e0 is 50, key is test_row_0/C:col10/1731332217678/Put/seqid=0 2024-11-11T13:36:59,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742240_1416 (size=12301) 2024-11-11T13:36:59,757 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T13:36:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:36:59,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. as already flushing 2024-11-11T13:36:59,851 INFO [Thread-1475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1475. 2024-11-11T13:36:59,851 DEBUG [Thread-1475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:59,851 DEBUG [Thread-1475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,851 DEBUG [Thread-1475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:59,855 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/fa31cfc730bd49448dcc72e64e9fa818 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/fa31cfc730bd49448dcc72e64e9fa818 2024-11-11T13:36:59,860 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e1cf571f2e70bf460349ad52744187e/A of 0e1cf571f2e70bf460349ad52744187e into fa31cfc730bd49448dcc72e64e9fa818(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:36:59,860 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:59,860 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., storeName=0e1cf571f2e70bf460349ad52744187e/A, priority=13, startTime=1731332219005; duration=0sec 2024-11-11T13:36:59,861 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:36:59,861 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e1cf571f2e70bf460349ad52744187e:A 2024-11-11T13:36:59,861 INFO [Thread-1477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1477. 2024-11-11T13:36:59,861 DEBUG [Thread-1477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:59,861 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,861 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:59,881 INFO [Thread-1473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1473. 2024-11-11T13:36:59,881 INFO [Thread-1479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1479. 2024-11-11T13:36:59,881 INFO [Thread-1471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1471. 2024-11-11T13:36:59,881 DEBUG [Thread-1473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:59,881 DEBUG [Thread-1479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:59,881 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,881 DEBUG [Thread-1471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:36:59,881 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,881 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,881 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,881 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,881 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:36:59,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:59,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:59,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:36:59,942 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/e0a3664f7bce4d1eae1efbf49ae837e0 2024-11-11T13:36:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/1424b05ec86c4de5bfb58259b774474d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/1424b05ec86c4de5bfb58259b774474d 2024-11-11T13:36:59,951 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/1424b05ec86c4de5bfb58259b774474d, entries=150, sequenceid=354, filesize=30.5 K 2024-11-11T13:36:59,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/bc02117414504409b9ed14a924e455a6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/bc02117414504409b9ed14a924e455a6 2024-11-11T13:36:59,955 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/bc02117414504409b9ed14a924e455a6, entries=150, sequenceid=354, filesize=12.0 K 2024-11-11T13:36:59,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/e0a3664f7bce4d1eae1efbf49ae837e0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e0a3664f7bce4d1eae1efbf49ae837e0 2024-11-11T13:36:59,959 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e0a3664f7bce4d1eae1efbf49ae837e0, entries=150, sequenceid=354, filesize=12.0 K 2024-11-11T13:36:59,960 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for 0e1cf571f2e70bf460349ad52744187e in 862ms, sequenceid=354, compaction requested=false 2024-11-11T13:36:59,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 0e1cf571f2e70bf460349ad52744187e: 2024-11-11T13:36:59,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:36:59,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-11T13:36:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-11T13:36:59,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T13:36:59,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9490 sec 2024-11-11T13:36:59,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.9550 sec 2024-11-11T13:37:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T13:37:00,160 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1157 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3467 rows 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1157 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3469 rows 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1154 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3461 rows 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1154 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3460 rows 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1142 2024-11-11T13:37:00,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3426 rows 2024-11-11T13:37:00,162 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:37:00,162 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:37:00,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:00,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:00,162 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:00,164 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T13:37:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T13:37:00,171 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332220171"}]},"ts":"1731332220171"} 2024-11-11T13:37:00,172 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T13:37:00,172 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T13:37:00,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T13:37:00,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, UNASSIGN}] 2024-11-11T13:37:00,174 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, UNASSIGN 2024-11-11T13:37:00,175 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=0e1cf571f2e70bf460349ad52744187e, regionState=CLOSING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:00,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, UNASSIGN because future has completed 2024-11-11T13:37:00,176 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T13:37:00,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:37:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T13:37:00,329 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:00,329 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T13:37:00,329 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing 0e1cf571f2e70bf460349ad52744187e, disabling compactions & flushes 2024-11-11T13:37:00,329 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:37:00,329 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:37:00,329 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. after waiting 0 ms 2024-11-11T13:37:00,329 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:37:00,329 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing 0e1cf571f2e70bf460349ad52744187e 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T13:37:00,337 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111103c0dbbb1fb54d8282c0e4eb962a66d5_0e1cf571f2e70bf460349ad52744187e is 50, key is test_row_0/A:col10/1731332219879/Put/seqid=0 2024-11-11T13:37:00,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742241_1417 (size=12454) 2024-11-11T13:37:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T13:37:00,741 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:00,744 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111103c0dbbb1fb54d8282c0e4eb962a66d5_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111103c0dbbb1fb54d8282c0e4eb962a66d5_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:00,745 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/7713777d6c8f4eb8833fd0fed5fe57f4, store: [table=TestAcidGuarantees family=A region=0e1cf571f2e70bf460349ad52744187e] 2024-11-11T13:37:00,746 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/7713777d6c8f4eb8833fd0fed5fe57f4 is 175, key is test_row_0/A:col10/1731332219879/Put/seqid=0 2024-11-11T13:37:00,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742242_1418 (size=31255) 2024-11-11T13:37:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T13:37:01,149 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/7713777d6c8f4eb8833fd0fed5fe57f4 2024-11-11T13:37:01,157 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/f318f7ab6620487b8d53fd67681b252e is 50, key is test_row_0/B:col10/1731332219879/Put/seqid=0 2024-11-11T13:37:01,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742243_1419 (size=12301) 2024-11-11T13:37:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T13:37:01,565 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/f318f7ab6620487b8d53fd67681b252e 2024-11-11T13:37:01,577 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/eec1ced034194928aecb558fe85874da is 50, key is test_row_0/C:col10/1731332219879/Put/seqid=0 2024-11-11T13:37:01,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742244_1420 (size=12301) 2024-11-11T13:37:01,981 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/eec1ced034194928aecb558fe85874da 2024-11-11T13:37:01,987 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/A/7713777d6c8f4eb8833fd0fed5fe57f4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/7713777d6c8f4eb8833fd0fed5fe57f4 2024-11-11T13:37:01,992 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/7713777d6c8f4eb8833fd0fed5fe57f4, entries=150, sequenceid=364, filesize=30.5 K 2024-11-11T13:37:01,994 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/B/f318f7ab6620487b8d53fd67681b252e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/f318f7ab6620487b8d53fd67681b252e 2024-11-11T13:37:01,997 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/f318f7ab6620487b8d53fd67681b252e, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T13:37:01,998 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/.tmp/C/eec1ced034194928aecb558fe85874da as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/eec1ced034194928aecb558fe85874da 2024-11-11T13:37:02,002 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/eec1ced034194928aecb558fe85874da, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T13:37:02,003 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0e1cf571f2e70bf460349ad52744187e in 1673ms, sequenceid=364, compaction requested=true 2024-11-11T13:37:02,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/8835673a4e334132afeddf1f2ebd5c1c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/325c27114ee243df9dea2f2f52ad8e66, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/f6696617db694689a81fa64b9c2cc2c4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/7caf30cde76341d0a9e2ceaf2bafeb31, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/089b5ab3c07745958dbed6fdee27c511, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/ead3a9c0a4fc4f8ab5f6810133402dca, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/4360b96d0fb3474e818eb37bbda79b7c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5b47a95e9bf47b1b932d998933bcf60, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/bbfc25537eb3417d8c546d359fcb4d1e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d03a0ac63d254c5f92c175debe48b13e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5886b21ae674c77b9b287e954d94397, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/44ce8ddeb4f749d4a939969291e00d62, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/b99ce59e18144826b542ce72054aa8f9, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/14b72c1989d54504b6ccff3348901083, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/a3d9862912514f2f85e7210d15f26753, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d67d5aa91f094e2a9c274037be98de40, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/5ea0f608ce334e38aeb889cf19fd65fd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2b0f142d00384e64b08fafaf1c24f2e0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/13dcb0a552db4358a662f88a6c7376d7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/0357276afe994da780b97dc442a61a8d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/da7b7ef9aac741299e030c53a4a4fc76, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2558ca9ada60401c8895a64f49b783ef] to archive 2024-11-11T13:37:02,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:37:02,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/8835673a4e334132afeddf1f2ebd5c1c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/8835673a4e334132afeddf1f2ebd5c1c 2024-11-11T13:37:02,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/325c27114ee243df9dea2f2f52ad8e66 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/325c27114ee243df9dea2f2f52ad8e66 2024-11-11T13:37:02,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/f6696617db694689a81fa64b9c2cc2c4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/f6696617db694689a81fa64b9c2cc2c4 2024-11-11T13:37:02,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/7caf30cde76341d0a9e2ceaf2bafeb31 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/7caf30cde76341d0a9e2ceaf2bafeb31 2024-11-11T13:37:02,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/089b5ab3c07745958dbed6fdee27c511 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/089b5ab3c07745958dbed6fdee27c511 2024-11-11T13:37:02,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/ead3a9c0a4fc4f8ab5f6810133402dca to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/ead3a9c0a4fc4f8ab5f6810133402dca 2024-11-11T13:37:02,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/4360b96d0fb3474e818eb37bbda79b7c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/4360b96d0fb3474e818eb37bbda79b7c 2024-11-11T13:37:02,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5b47a95e9bf47b1b932d998933bcf60 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5b47a95e9bf47b1b932d998933bcf60 2024-11-11T13:37:02,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/bbfc25537eb3417d8c546d359fcb4d1e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/bbfc25537eb3417d8c546d359fcb4d1e 2024-11-11T13:37:02,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d03a0ac63d254c5f92c175debe48b13e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d03a0ac63d254c5f92c175debe48b13e 2024-11-11T13:37:02,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5886b21ae674c77b9b287e954d94397 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d5886b21ae674c77b9b287e954d94397 2024-11-11T13:37:02,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/44ce8ddeb4f749d4a939969291e00d62 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/44ce8ddeb4f749d4a939969291e00d62 2024-11-11T13:37:02,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/b99ce59e18144826b542ce72054aa8f9 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/b99ce59e18144826b542ce72054aa8f9 2024-11-11T13:37:02,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/14b72c1989d54504b6ccff3348901083 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/14b72c1989d54504b6ccff3348901083 2024-11-11T13:37:02,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/a3d9862912514f2f85e7210d15f26753 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/a3d9862912514f2f85e7210d15f26753 2024-11-11T13:37:02,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d67d5aa91f094e2a9c274037be98de40 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/d67d5aa91f094e2a9c274037be98de40 2024-11-11T13:37:02,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/5ea0f608ce334e38aeb889cf19fd65fd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/5ea0f608ce334e38aeb889cf19fd65fd 2024-11-11T13:37:02,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2b0f142d00384e64b08fafaf1c24f2e0 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2b0f142d00384e64b08fafaf1c24f2e0 2024-11-11T13:37:02,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/13dcb0a552db4358a662f88a6c7376d7 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/13dcb0a552db4358a662f88a6c7376d7 2024-11-11T13:37:02,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/0357276afe994da780b97dc442a61a8d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/0357276afe994da780b97dc442a61a8d 2024-11-11T13:37:02,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/da7b7ef9aac741299e030c53a4a4fc76 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/da7b7ef9aac741299e030c53a4a4fc76 2024-11-11T13:37:02,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2558ca9ada60401c8895a64f49b783ef to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/2558ca9ada60401c8895a64f49b783ef 2024-11-11T13:37:02,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/3d9e39b8926b47f8961dd6687ffafb7b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/83ae8d8f6a094209a1a7c2afc3f16402, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/852b0a180e4442808a5dc1c613a9591a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/375148c14e324000b131601cdedb4a1f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/685c6fbaa0964de98e969cfddc737391, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/866627ee5e8f484993e7a9b6389da37e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/576a2a0f7dee45c2b57b4c8de624c664, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/484aaf131cd642ba953c1beb20ec6816, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/f91ad12b75b64cd8bb042bb3fa480f08, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8e2f00e02d5a46ecb3ead36997e3a2c7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/432d491b488640bb9fc61d87132ef739, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8ee0b0bd09ee4faaafb299935abd5d93, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/1dd7f59ac81844a383cc01d472e623d5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/1f56679b6cc945348b9b14e70d181b4a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c1cd694dcd743b8980a44141aba3335, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/69b32a34f4b64d30b1ba565da883d765, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/9dca03836206461ab2fa9bfbf7b76d18, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c14e6a00dc24057a65f8a815b9eb779, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/eb3b569dcc514a18bcae6320e9bfc703, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/84ea751b10884ae1a2766645d752fa57, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/78cd6a3b250c4650aba46ab53a95f681, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/36a670abcb244ac48db41d0d59c2f626] to archive 2024-11-11T13:37:02,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:37:02,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/3d9e39b8926b47f8961dd6687ffafb7b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/3d9e39b8926b47f8961dd6687ffafb7b 2024-11-11T13:37:02,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/83ae8d8f6a094209a1a7c2afc3f16402 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/83ae8d8f6a094209a1a7c2afc3f16402 2024-11-11T13:37:02,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/852b0a180e4442808a5dc1c613a9591a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/852b0a180e4442808a5dc1c613a9591a 2024-11-11T13:37:02,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/375148c14e324000b131601cdedb4a1f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/375148c14e324000b131601cdedb4a1f 2024-11-11T13:37:02,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/685c6fbaa0964de98e969cfddc737391 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/685c6fbaa0964de98e969cfddc737391 2024-11-11T13:37:02,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/866627ee5e8f484993e7a9b6389da37e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/866627ee5e8f484993e7a9b6389da37e 2024-11-11T13:37:02,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/576a2a0f7dee45c2b57b4c8de624c664 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/576a2a0f7dee45c2b57b4c8de624c664 2024-11-11T13:37:02,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/484aaf131cd642ba953c1beb20ec6816 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/484aaf131cd642ba953c1beb20ec6816 2024-11-11T13:37:02,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/f91ad12b75b64cd8bb042bb3fa480f08 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/f91ad12b75b64cd8bb042bb3fa480f08 2024-11-11T13:37:02,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8e2f00e02d5a46ecb3ead36997e3a2c7 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8e2f00e02d5a46ecb3ead36997e3a2c7 2024-11-11T13:37:02,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/432d491b488640bb9fc61d87132ef739 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/432d491b488640bb9fc61d87132ef739 2024-11-11T13:37:02,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8ee0b0bd09ee4faaafb299935abd5d93 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/8ee0b0bd09ee4faaafb299935abd5d93 2024-11-11T13:37:02,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/1dd7f59ac81844a383cc01d472e623d5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/1dd7f59ac81844a383cc01d472e623d5 2024-11-11T13:37:02,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/1f56679b6cc945348b9b14e70d181b4a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/1f56679b6cc945348b9b14e70d181b4a 2024-11-11T13:37:02,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c1cd694dcd743b8980a44141aba3335 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c1cd694dcd743b8980a44141aba3335 2024-11-11T13:37:02,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/69b32a34f4b64d30b1ba565da883d765 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/69b32a34f4b64d30b1ba565da883d765 2024-11-11T13:37:02,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/9dca03836206461ab2fa9bfbf7b76d18 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/9dca03836206461ab2fa9bfbf7b76d18 2024-11-11T13:37:02,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c14e6a00dc24057a65f8a815b9eb779 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/2c14e6a00dc24057a65f8a815b9eb779 2024-11-11T13:37:02,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/eb3b569dcc514a18bcae6320e9bfc703 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/eb3b569dcc514a18bcae6320e9bfc703 2024-11-11T13:37:02,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/84ea751b10884ae1a2766645d752fa57 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/84ea751b10884ae1a2766645d752fa57 2024-11-11T13:37:02,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/78cd6a3b250c4650aba46ab53a95f681 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/78cd6a3b250c4650aba46ab53a95f681 2024-11-11T13:37:02,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/36a670abcb244ac48db41d0d59c2f626 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/36a670abcb244ac48db41d0d59c2f626 2024-11-11T13:37:02,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95055f29a6524892b9a3034854e07f65, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/60e358f9d2784675b4c5c64f9e716393, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/0e60d6e170c248849dccd64abc5fcdc8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e240066dcdc947ce99b07f2545038b4d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/64c82e3afd9a47ad93ebc002175972bf, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/78edfc0cf76d4654a080aaf5e81896a6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/ac6ee2b81454460899ce129ea69cff19, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/7ef61101550d4c0ca30cbe6acc50fc75, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/4498b617b2784352a03a7645df1027cb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95016652bcc34ee4912f46ea37ee0f68, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/4cc102a8782a44ffbe76fc96174ed426, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/04eee5e917354baea72b3c933f0e98ac, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/c6a5f099101a4119bddbaff00a6c46eb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/03e4a838d2b746119a63934aa42d8ea3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e4a75ebe523c4911a684a8ee599585a2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/9207c0a88475406ba4a45490daf17b6e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/6b74633f935e42d8a8b3ca82695bf382, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/93baa6a5bd55490785d7ec302404f5a4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/f4267674bc2b45a583da5a6bb11c6dfd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/2953daebac79472a98a9921c1ca27fac, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/f3e3be9fb50d43949c6b3d6d436a3aed, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/1423b92008174b68b5ff2a4080e5a4cd] to archive 2024-11-11T13:37:02,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:37:02,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95055f29a6524892b9a3034854e07f65 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95055f29a6524892b9a3034854e07f65 2024-11-11T13:37:02,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/60e358f9d2784675b4c5c64f9e716393 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/60e358f9d2784675b4c5c64f9e716393 2024-11-11T13:37:02,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/0e60d6e170c248849dccd64abc5fcdc8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/0e60d6e170c248849dccd64abc5fcdc8 2024-11-11T13:37:02,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e240066dcdc947ce99b07f2545038b4d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e240066dcdc947ce99b07f2545038b4d 2024-11-11T13:37:02,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/64c82e3afd9a47ad93ebc002175972bf to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/64c82e3afd9a47ad93ebc002175972bf 2024-11-11T13:37:02,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/78edfc0cf76d4654a080aaf5e81896a6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/78edfc0cf76d4654a080aaf5e81896a6 2024-11-11T13:37:02,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/ac6ee2b81454460899ce129ea69cff19 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/ac6ee2b81454460899ce129ea69cff19 2024-11-11T13:37:02,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/7ef61101550d4c0ca30cbe6acc50fc75 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/7ef61101550d4c0ca30cbe6acc50fc75 2024-11-11T13:37:02,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/4498b617b2784352a03a7645df1027cb to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/4498b617b2784352a03a7645df1027cb 2024-11-11T13:37:02,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95016652bcc34ee4912f46ea37ee0f68 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/95016652bcc34ee4912f46ea37ee0f68 2024-11-11T13:37:02,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/4cc102a8782a44ffbe76fc96174ed426 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/4cc102a8782a44ffbe76fc96174ed426 2024-11-11T13:37:02,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/04eee5e917354baea72b3c933f0e98ac to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/04eee5e917354baea72b3c933f0e98ac 2024-11-11T13:37:02,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/c6a5f099101a4119bddbaff00a6c46eb to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/c6a5f099101a4119bddbaff00a6c46eb 2024-11-11T13:37:02,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/03e4a838d2b746119a63934aa42d8ea3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/03e4a838d2b746119a63934aa42d8ea3 2024-11-11T13:37:02,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e4a75ebe523c4911a684a8ee599585a2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e4a75ebe523c4911a684a8ee599585a2 2024-11-11T13:37:02,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/9207c0a88475406ba4a45490daf17b6e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/9207c0a88475406ba4a45490daf17b6e 2024-11-11T13:37:02,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/6b74633f935e42d8a8b3ca82695bf382 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/6b74633f935e42d8a8b3ca82695bf382 2024-11-11T13:37:02,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/93baa6a5bd55490785d7ec302404f5a4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/93baa6a5bd55490785d7ec302404f5a4 2024-11-11T13:37:02,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/f4267674bc2b45a583da5a6bb11c6dfd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/f4267674bc2b45a583da5a6bb11c6dfd 2024-11-11T13:37:02,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/2953daebac79472a98a9921c1ca27fac to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/2953daebac79472a98a9921c1ca27fac 2024-11-11T13:37:02,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/f3e3be9fb50d43949c6b3d6d436a3aed to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/f3e3be9fb50d43949c6b3d6d436a3aed 2024-11-11T13:37:02,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/1423b92008174b68b5ff2a4080e5a4cd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/1423b92008174b68b5ff2a4080e5a4cd 2024-11-11T13:37:02,082 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/recovered.edits/367.seqid, newMaxSeqId=367, maxSeqId=4 2024-11-11T13:37:02,082 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. 2024-11-11T13:37:02,082 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for 0e1cf571f2e70bf460349ad52744187e: Waiting for close lock at 1731332220329Running coprocessor pre-close hooks at 1731332220329Disabling compacts and flushes for region at 1731332220329Disabling writes for close at 1731332220329Obtaining lock to block concurrent updates at 1731332220329Preparing flush snapshotting stores in 0e1cf571f2e70bf460349ad52744187e at 1731332220329Finished memstore snapshotting TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731332220330 (+1 ms)Flushing stores of TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e. at 1731332220330Flushing 0e1cf571f2e70bf460349ad52744187e/A: creating writer at 1731332220331 (+1 ms)Flushing mob file 0e1cf571f2e70bf460349ad52744187e/A: appending metadata at 1731332220337 (+6 ms)Flushing mob file 0e1cf571f2e70bf460349ad52744187e/A: closing flushed file at 1731332220337Flushing 0e1cf571f2e70bf460349ad52744187e/A: appending metadata at 1731332220745 (+408 ms)Flushing 0e1cf571f2e70bf460349ad52744187e/A: closing flushed file at 1731332220745Flushing 0e1cf571f2e70bf460349ad52744187e/B: creating writer at 1731332221154 (+409 ms)Flushing 0e1cf571f2e70bf460349ad52744187e/B: appending metadata at 1731332221157 (+3 ms)Flushing 0e1cf571f2e70bf460349ad52744187e/B: closing flushed file at 1731332221157Flushing 0e1cf571f2e70bf460349ad52744187e/C: creating writer at 1731332221573 (+416 ms)Flushing 0e1cf571f2e70bf460349ad52744187e/C: appending metadata at 1731332221577 (+4 ms)Flushing 0e1cf571f2e70bf460349ad52744187e/C: closing flushed file at 1731332221577Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56f45fb6: reopening flushed file at 1731332221986 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61c9007e: reopening flushed file at 1731332221993 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42110a5a: reopening flushed file at 1731332221998 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0e1cf571f2e70bf460349ad52744187e in 1673ms, sequenceid=364, compaction requested=true at 1731332222003 (+5 ms)Writing region close event to WAL at 1731332222079 (+76 ms)Running coprocessor post-close hooks at 1731332222082 (+3 ms)Closed at 1731332222082 2024-11-11T13:37:02,084 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed 0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,084 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=0e1cf571f2e70bf460349ad52744187e, regionState=CLOSED 2024-11-11T13:37:02,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:37:02,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-11T13:37:02,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure 0e1cf571f2e70bf460349ad52744187e, server=37d26ecf6460,36877,1731332101576 in 1.9100 sec 2024-11-11T13:37:02,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-11T13:37:02,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e1cf571f2e70bf460349ad52744187e, UNASSIGN in 1.9130 sec 2024-11-11T13:37:02,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T13:37:02,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9180 sec 2024-11-11T13:37:02,093 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332222093"}]},"ts":"1731332222093"} 2024-11-11T13:37:02,094 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T13:37:02,094 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T13:37:02,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9310 sec 2024-11-11T13:37:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T13:37:02,320 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:02,320 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T13:37:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:02,323 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T13:37:02,324 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:02,326 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,330 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/recovered.edits] 2024-11-11T13:37:02,333 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/1424b05ec86c4de5bfb58259b774474d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/1424b05ec86c4de5bfb58259b774474d 2024-11-11T13:37:02,334 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/7713777d6c8f4eb8833fd0fed5fe57f4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/7713777d6c8f4eb8833fd0fed5fe57f4 2024-11-11T13:37:02,335 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/fa31cfc730bd49448dcc72e64e9fa818 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/A/fa31cfc730bd49448dcc72e64e9fa818 2024-11-11T13:37:02,337 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/7c7e5bd8eb934b07afb36a86fcf830d8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/7c7e5bd8eb934b07afb36a86fcf830d8 2024-11-11T13:37:02,339 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/bc02117414504409b9ed14a924e455a6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/bc02117414504409b9ed14a924e455a6 2024-11-11T13:37:02,340 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/f318f7ab6620487b8d53fd67681b252e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/B/f318f7ab6620487b8d53fd67681b252e 2024-11-11T13:37:02,342 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/c4bbf19011ed49bb89da2ddf78b2d5a5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/c4bbf19011ed49bb89da2ddf78b2d5a5 2024-11-11T13:37:02,343 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e0a3664f7bce4d1eae1efbf49ae837e0 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/e0a3664f7bce4d1eae1efbf49ae837e0 2024-11-11T13:37:02,344 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/eec1ced034194928aecb558fe85874da to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/C/eec1ced034194928aecb558fe85874da 2024-11-11T13:37:02,345 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/recovered.edits/367.seqid to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e/recovered.edits/367.seqid 2024-11-11T13:37:02,345 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,346 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T13:37:02,346 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T13:37:02,346 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T13:37:02,348 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111103c0dbbb1fb54d8282c0e4eb962a66d5_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111103c0dbbb1fb54d8282c0e4eb962a66d5_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,349 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110f8d2eb4f41e47a3a42ea8eaf2db5655_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110f8d2eb4f41e47a3a42ea8eaf2db5655_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,349 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111d839b2e92a84a1c99ef74edb1005f48_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111d839b2e92a84a1c99ef74edb1005f48_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,350 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111e7f24d37b374c9b86f94c284730fc20_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111e7f24d37b374c9b86f94c284730fc20_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,351 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111122f050aeaaca4a28950166e1cfae1212_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111122f050aeaaca4a28950166e1cfae1212_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,352 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113b9b1dd9c0aa4f3db9d34b46b2d2524c_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113b9b1dd9c0aa4f3db9d34b46b2d2524c_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,352 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113bd7fa7118464699a43b5e78179479e9_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113bd7fa7118464699a43b5e78179479e9_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,353 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113dbdfd080e1e4b8ca91e0d3cf0c62fbb_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113dbdfd080e1e4b8ca91e0d3cf0c62fbb_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,354 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111142d8e8a9b3f04ab3a27a9ec5cb4be714_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111142d8e8a9b3f04ab3a27a9ec5cb4be714_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,355 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111611427ccf3b94726a67e612bd5ad7fab_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111611427ccf3b94726a67e612bd5ad7fab_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,356 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111163d74662695e4ec180b8635b168e2d0d_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111163d74662695e4ec180b8635b168e2d0d_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,357 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111166ff4256092341a29776aba2c09d7939_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111166ff4256092341a29776aba2c09d7939_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,358 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116ece6801e28040818c570c7eeab5e7d2_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116ece6801e28040818c570c7eeab5e7d2_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,359 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111789d207bbaae48aebd61f7d403aae078_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111789d207bbaae48aebd61f7d403aae078_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,360 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118101e4f78ff64cdba0b5f3039c2381b9_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118101e4f78ff64cdba0b5f3039c2381b9_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,361 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119c6c361e528d4fc6933d392f8653d78e_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119c6c361e528d4fc6933d392f8653d78e_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,361 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cf6b3faac47f4a26b60cc05445d14bd7_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cf6b3faac47f4a26b60cc05445d14bd7_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,362 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f8177faac1424444b49ffc08711b45f9_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f8177faac1424444b49ffc08711b45f9_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,363 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f9e7f7c260ad4a5390e0115e13ea72a8_0e1cf571f2e70bf460349ad52744187e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f9e7f7c260ad4a5390e0115e13ea72a8_0e1cf571f2e70bf460349ad52744187e 2024-11-11T13:37:02,363 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T13:37:02,365 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:02,367 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T13:37:02,369 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T13:37:02,370 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:02,370 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T13:37:02,370 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731332222370"}]},"ts":"9223372036854775807"} 2024-11-11T13:37:02,372 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T13:37:02,372 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0e1cf571f2e70bf460349ad52744187e, NAME => 'TestAcidGuarantees,,1731332196848.0e1cf571f2e70bf460349ad52744187e.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T13:37:02,372 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T13:37:02,372 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731332222372"}]},"ts":"9223372036854775807"} 2024-11-11T13:37:02,373 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T13:37:02,374 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:02,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-11T13:37:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T13:37:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T13:37:02,429 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:02,437 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 236) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_567230531_22 at /127.0.0.1:47302 [Waiting for operation #945] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-931856926_22 at /127.0.0.1:43062 [Waiting for operation #775] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-931856926_22 at /127.0.0.1:47534 [Waiting for operation #876] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_567230531_22 at /127.0.0.1:43052 [Waiting for operation #793] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=577 (was 622), ProcessCount=11 (was 11), AvailableMemoryMB=2738 (was 2904) 2024-11-11T13:37:02,445 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=577, ProcessCount=11, AvailableMemoryMB=2738 2024-11-11T13:37:02,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T13:37:02,447 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:37:02,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:02,448 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T13:37:02,449 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:02,449 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-11-11T13:37:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-11T13:37:02,449 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T13:37:02,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742245_1421 (size=959) 2024-11-11T13:37:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-11T13:37:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-11T13:37:02,856 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92 2024-11-11T13:37:02,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742246_1422 (size=53) 2024-11-11T13:37:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-11T13:37:03,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:37:03,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ed96da846868b1163b5c13dff6f86179, disabling compactions & flushes 2024-11-11T13:37:03,262 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:03,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:03,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. after waiting 0 ms 2024-11-11T13:37:03,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:03,262 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:03,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed96da846868b1163b5c13dff6f86179: Waiting for close lock at 1731332223262Disabling compacts and flushes for region at 1731332223262Disabling writes for close at 1731332223262Writing region close event to WAL at 1731332223262Closed at 1731332223262 2024-11-11T13:37:03,263 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T13:37:03,264 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731332223263"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731332223263"}]},"ts":"1731332223263"} 2024-11-11T13:37:03,265 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T13:37:03,266 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T13:37:03,266 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332223266"}]},"ts":"1731332223266"} 2024-11-11T13:37:03,268 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T13:37:03,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed96da846868b1163b5c13dff6f86179, ASSIGN}] 2024-11-11T13:37:03,270 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed96da846868b1163b5c13dff6f86179, ASSIGN 2024-11-11T13:37:03,271 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed96da846868b1163b5c13dff6f86179, ASSIGN; state=OFFLINE, location=37d26ecf6460,36877,1731332101576; forceNewPlan=false, retain=false 2024-11-11T13:37:03,421 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=ed96da846868b1163b5c13dff6f86179, regionState=OPENING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:03,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed96da846868b1163b5c13dff6f86179, ASSIGN because future has completed 2024-11-11T13:37:03,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:37:03,581 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:03,582 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:37:03,582 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,582 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:37:03,582 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,582 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,583 INFO [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,585 INFO [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed96da846868b1163b5c13dff6f86179 columnFamilyName A 2024-11-11T13:37:03,585 DEBUG [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:03,587 INFO [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] regionserver.HStore(327): Store=ed96da846868b1163b5c13dff6f86179/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:37:03,587 INFO [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-11T13:37:03,592 INFO [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed96da846868b1163b5c13dff6f86179 columnFamilyName B 2024-11-11T13:37:03,592 DEBUG [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:03,593 INFO [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] regionserver.HStore(327): Store=ed96da846868b1163b5c13dff6f86179/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:37:03,593 INFO [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,594 INFO [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed96da846868b1163b5c13dff6f86179 columnFamilyName C 2024-11-11T13:37:03,594 DEBUG [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:03,595 INFO [StoreOpener-ed96da846868b1163b5c13dff6f86179-1 {}] regionserver.HStore(327): Store=ed96da846868b1163b5c13dff6f86179/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:37:03,595 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,595 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,596 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,596 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,596 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,603 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T13:37:03,605 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,607 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:37:03,609 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened ed96da846868b1163b5c13dff6f86179; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72149368, jitterRate=0.07510936260223389}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T13:37:03,609 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:03,610 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for ed96da846868b1163b5c13dff6f86179: Running coprocessor pre-open hook at 1731332223582Writing region info on filesystem at 1731332223582Initializing all the Stores at 1731332223583 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332223583Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332223583Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332223583Cleaning up temporary data from old regions at 1731332223596 (+13 ms)Running coprocessor post-open hooks at 1731332223609 (+13 ms)Region opened successfully at 1731332223610 (+1 ms) 2024-11-11T13:37:03,610 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., pid=108, masterSystemTime=1731332223578 2024-11-11T13:37:03,612 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:03,612 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:03,613 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=ed96da846868b1163b5c13dff6f86179, regionState=OPEN, openSeqNum=2, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:03,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:37:03,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-11T13:37:03,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 in 192 msec 2024-11-11T13:37:03,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-11T13:37:03,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed96da846868b1163b5c13dff6f86179, ASSIGN in 348 msec 2024-11-11T13:37:03,618 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T13:37:03,619 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332223618"}]},"ts":"1731332223618"} 2024-11-11T13:37:03,620 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T13:37:03,621 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T13:37:03,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-11T13:37:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-11T13:37:04,610 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:04,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd73b98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:04,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:04,613 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f68ebb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:04,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:04,616 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72166306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,617 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:04,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:04,622 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T13:37:04,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 37d26ecf6460,35609,1731332100707 2024-11-11T13:37:04,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38ceb36e 2024-11-11T13:37:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T13:37:04,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T13:37:04,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f5cac0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:04,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:04,627 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60a00b3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:04,628 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40370d2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d436c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:04,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:04,630 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@529fb07f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:04,631 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5e1d04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfb5c2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:04,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:04,632 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6954397, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:04,634 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:04,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78cc2c82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1042326a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:04,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:04,635 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d8f0031, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:04,637 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a0fe5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac88004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:04,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:04,639 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:04,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:04,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:04,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c705773, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:04,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:04,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:04,641 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:04,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c1490c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b072e4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:04,642 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:04,643 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:04,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:04,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:04,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30cd521e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:04,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:04,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:04,644 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:04,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633d226f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738f6f36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:04,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:04,646 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1020fd7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:04,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:04,648 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4a9def, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f90f6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:04,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:04,650 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:04,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc53ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:04,652 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:04,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9f1b0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14bd514d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:04,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:04,653 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:04,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6200eb81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:04,654 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df68577, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffa0ad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:04,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:04,656 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:04,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:04,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:04,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d3ea0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:04,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:04,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:04,657 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fd53105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:04,658 DEBUG [Thread-1910 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:04,658 DEBUG [Thread-1914 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:04,658 DEBUG [Thread-1921 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:04,658 DEBUG [Thread-1908 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:04,658 DEBUG [Thread-1912 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:04,658 DEBUG [Thread-1923 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:04,659 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:04,659 DEBUG [Thread-1916 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:04,659 DEBUG [Thread-1925 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:04,660 DEBUG [Thread-1908 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-11T13:37:04,660 DEBUG [Thread-1908 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:04,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:04,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:04,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:04,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:04,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:04,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T13:37:04,661 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:04,661 DEBUG [Thread-1927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:04,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:04,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:04,662 DEBUG [Thread-1919 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:37:04,662 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:04,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:04,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:37:04,663 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:37:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:37:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:37:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:37:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:37:04,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:04,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:37:04,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:37:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2] 2024-11-11T13:37:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:04,672 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:04,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:37:04,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/9f39ff289f9e42319bc844b91b4160f1 is 50, key is test_row_0/A:col10/1731332224664/Put/seqid=0 2024-11-11T13:37:04,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742247_1423 (size=16681) 2024-11-11T13:37:04,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/9f39ff289f9e42319bc844b91b4160f1 2024-11-11T13:37:04,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/bc9a3cd40ecb4997a6b19c35c1857702 is 50, key is test_row_0/B:col10/1731332224664/Put/seqid=0 2024-11-11T13:37:04,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742248_1424 (size=12001) 2024-11-11T13:37:04,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332284713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:04,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332284714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:04,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332284714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332284715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332284715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:04,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T13:37:04,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T13:37:04,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:04,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:04,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:04,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:04,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:04,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:04,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332284830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332284829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332284830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332284830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:04,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332284830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:04,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:04,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T13:37:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:04,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T13:37:05,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332285039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332285039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332285040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332285040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332285040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T13:37:05,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:05,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:05,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:05,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:05,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:05,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/bc9a3cd40ecb4997a6b19c35c1857702 2024-11-11T13:37:05,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/49566b23b50d491896454693f0a55fee is 50, key is test_row_0/C:col10/1731332224664/Put/seqid=0 2024-11-11T13:37:05,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742249_1425 (size=12001) 2024-11-11T13:37:05,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T13:37:05,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:05,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:05,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:05,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:05,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T13:37:05,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332285349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332285349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332285349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332285349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332285349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T13:37:05,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:05,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:05,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:05,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:05,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:05,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/49566b23b50d491896454693f0a55fee 2024-11-11T13:37:05,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/9f39ff289f9e42319bc844b91b4160f1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/9f39ff289f9e42319bc844b91b4160f1 2024-11-11T13:37:05,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/9f39ff289f9e42319bc844b91b4160f1, entries=250, sequenceid=13, filesize=16.3 K 2024-11-11T13:37:05,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/bc9a3cd40ecb4997a6b19c35c1857702 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/bc9a3cd40ecb4997a6b19c35c1857702 2024-11-11T13:37:05,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/bc9a3cd40ecb4997a6b19c35c1857702, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T13:37:05,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/49566b23b50d491896454693f0a55fee as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49566b23b50d491896454693f0a55fee 2024-11-11T13:37:05,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49566b23b50d491896454693f0a55fee, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T13:37:05,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for ed96da846868b1163b5c13dff6f86179 in 896ms, sequenceid=13, compaction requested=false 2024-11-11T13:37:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:05,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T13:37:05,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:05,580 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:37:05,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/22b6d17050b94e1c9a4d585ef7fdfc72 is 50, key is test_row_0/A:col10/1731332224714/Put/seqid=0 2024-11-11T13:37:05,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742250_1426 (size=12001) 2024-11-11T13:37:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T13:37:05,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:05,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332285864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332285865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332285866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332285867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332285871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332285980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332285980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332285980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332285980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332285980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:05,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:05,993 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/22b6d17050b94e1c9a4d585ef7fdfc72 2024-11-11T13:37:06,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/604934cc2c9a464eb3871479dc4b5131 is 50, key is test_row_0/B:col10/1731332224714/Put/seqid=0 2024-11-11T13:37:06,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742251_1427 (size=12001) 2024-11-11T13:37:06,006 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/604934cc2c9a464eb3871479dc4b5131 2024-11-11T13:37:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/a2f5603dbe9348e1bac007c61ddb133a is 50, key is test_row_0/C:col10/1731332224714/Put/seqid=0 2024-11-11T13:37:06,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742252_1428 (size=12001) 2024-11-11T13:37:06,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332286190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332286190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332286190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332286190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332286190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,416 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/a2f5603dbe9348e1bac007c61ddb133a 2024-11-11T13:37:06,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/22b6d17050b94e1c9a4d585ef7fdfc72 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/22b6d17050b94e1c9a4d585ef7fdfc72 2024-11-11T13:37:06,423 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/22b6d17050b94e1c9a4d585ef7fdfc72, entries=150, sequenceid=38, filesize=11.7 K 2024-11-11T13:37:06,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/604934cc2c9a464eb3871479dc4b5131 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/604934cc2c9a464eb3871479dc4b5131 2024-11-11T13:37:06,426 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/604934cc2c9a464eb3871479dc4b5131, entries=150, sequenceid=38, filesize=11.7 K 2024-11-11T13:37:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/a2f5603dbe9348e1bac007c61ddb133a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/a2f5603dbe9348e1bac007c61ddb133a 2024-11-11T13:37:06,430 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/a2f5603dbe9348e1bac007c61ddb133a, entries=150, sequenceid=38, filesize=11.7 K 2024-11-11T13:37:06,430 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ed96da846868b1163b5c13dff6f86179 in 850ms, sequenceid=38, compaction requested=false 2024-11-11T13:37:06,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:06,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:06,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-11T13:37:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-11T13:37:06,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T13:37:06,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7700 sec 2024-11-11T13:37:06,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.7750 sec 2024-11-11T13:37:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:06,501 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:06,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T13:37:06,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/1d1fe98a2fb24a0c8110052206b4b855 is 50, key is test_row_0/A:col10/1731332226501/Put/seqid=0 2024-11-11T13:37:06,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742253_1429 (size=16681) 2024-11-11T13:37:06,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332286532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332286532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332286533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332286533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332286538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332286649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332286649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332286649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332286649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332286650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T13:37:06,810 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:06,810 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-11T13:37:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T13:37:06,812 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:06,812 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:06,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332286859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332286860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332286860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332286860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332286860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:06,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/1d1fe98a2fb24a0c8110052206b4b855 2024-11-11T13:37:06,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a108c7697a724599a6511fabdbf6025a is 50, key is test_row_0/B:col10/1731332226501/Put/seqid=0 2024-11-11T13:37:06,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742254_1430 (size=12001) 2024-11-11T13:37:06,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T13:37:06,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T13:37:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:06,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T13:37:07,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:07,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T13:37:07,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332287170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:07,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332287170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:07,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:07,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332287170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:07,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332287179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:07,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332287179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:07,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T13:37:07,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:07,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a108c7697a724599a6511fabdbf6025a 2024-11-11T13:37:07,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/49e5ec4b34b04e8f9163c8966746c7e2 is 50, key is test_row_0/C:col10/1731332226501/Put/seqid=0 2024-11-11T13:37:07,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742255_1431 (size=12001) 2024-11-11T13:37:07,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T13:37:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T13:37:07,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T13:37:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,606 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T13:37:07,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332287689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:07,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:07,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332287689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:07,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:07,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332287689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:07,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:07,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:07,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332287699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:07,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:07,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332287699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:07,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:07,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:07,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T13:37:07,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/49e5ec4b34b04e8f9163c8966746c7e2 2024-11-11T13:37:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:07,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/1d1fe98a2fb24a0c8110052206b4b855 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1d1fe98a2fb24a0c8110052206b4b855 2024-11-11T13:37:07,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1d1fe98a2fb24a0c8110052206b4b855, entries=250, sequenceid=51, filesize=16.3 K 2024-11-11T13:37:07,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a108c7697a724599a6511fabdbf6025a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a108c7697a724599a6511fabdbf6025a 2024-11-11T13:37:07,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a108c7697a724599a6511fabdbf6025a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T13:37:07,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/49e5ec4b34b04e8f9163c8966746c7e2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49e5ec4b34b04e8f9163c8966746c7e2 2024-11-11T13:37:07,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49e5ec4b34b04e8f9163c8966746c7e2, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T13:37:07,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ed96da846868b1163b5c13dff6f86179 in 1258ms, sequenceid=51, compaction requested=true 2024-11-11T13:37:07,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:07,760 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:07,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:07,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:07,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:07,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:07,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:07,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:37:07,763 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:07,772 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:07,772 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:07,772 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,772 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/bc9a3cd40ecb4997a6b19c35c1857702, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/604934cc2c9a464eb3871479dc4b5131, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a108c7697a724599a6511fabdbf6025a] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=35.2 K 2024-11-11T13:37:07,775 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:07,775 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:07,775 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,775 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/9f39ff289f9e42319bc844b91b4160f1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/22b6d17050b94e1c9a4d585ef7fdfc72, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1d1fe98a2fb24a0c8110052206b4b855] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=44.3 K 2024-11-11T13:37:07,776 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc9a3cd40ecb4997a6b19c35c1857702, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731332224664 2024-11-11T13:37:07,777 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f39ff289f9e42319bc844b91b4160f1, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731332224664 2024-11-11T13:37:07,777 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 604934cc2c9a464eb3871479dc4b5131, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731332224710 2024-11-11T13:37:07,777 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b6d17050b94e1c9a4d585ef7fdfc72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731332224710 2024-11-11T13:37:07,777 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a108c7697a724599a6511fabdbf6025a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731332225870 2024-11-11T13:37:07,783 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d1fe98a2fb24a0c8110052206b4b855, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731332225863 2024-11-11T13:37:07,787 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:07,787 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/8f24b5023eb7409bab408ad0928a61bd is 50, key is test_row_0/B:col10/1731332226501/Put/seqid=0 2024-11-11T13:37:07,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742256_1432 (size=12104) 2024-11-11T13:37:07,803 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:07,803 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/43e1c22bc78d467ca0ab090520cf97bc is 50, key is test_row_0/A:col10/1731332226501/Put/seqid=0 2024-11-11T13:37:07,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742257_1433 (size=12104) 2024-11-11T13:37:07,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T13:37:07,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:07,883 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T13:37:07,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/2d6b55b8c5e84d2ab51957095a96c5c4 is 50, key is test_row_0/A:col10/1731332226537/Put/seqid=0 2024-11-11T13:37:07,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742258_1434 (size=12001) 2024-11-11T13:37:07,904 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/2d6b55b8c5e84d2ab51957095a96c5c4 2024-11-11T13:37:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/71d13b719ce34acca8f194134517934c is 50, key is test_row_0/B:col10/1731332226537/Put/seqid=0 2024-11-11T13:37:07,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742259_1435 (size=12001) 2024-11-11T13:37:07,920 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/71d13b719ce34acca8f194134517934c 2024-11-11T13:37:07,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/e048c3329bd546c08234f8d769781238 is 50, key is test_row_0/C:col10/1731332226537/Put/seqid=0 2024-11-11T13:37:07,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742260_1436 (size=12001) 2024-11-11T13:37:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T13:37:08,219 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/8f24b5023eb7409bab408ad0928a61bd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8f24b5023eb7409bab408ad0928a61bd 2024-11-11T13:37:08,234 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into 8f24b5023eb7409bab408ad0928a61bd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:08,234 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:08,234 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=13, startTime=1731332227760; duration=0sec 2024-11-11T13:37:08,234 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:08,234 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:08,234 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:08,237 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:08,237 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:08,237 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:08,238 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49566b23b50d491896454693f0a55fee, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/a2f5603dbe9348e1bac007c61ddb133a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49e5ec4b34b04e8f9163c8966746c7e2] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=35.2 K 2024-11-11T13:37:08,240 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49566b23b50d491896454693f0a55fee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731332224664 2024-11-11T13:37:08,241 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f5603dbe9348e1bac007c61ddb133a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731332224710 2024-11-11T13:37:08,241 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49e5ec4b34b04e8f9163c8966746c7e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731332225870 2024-11-11T13:37:08,253 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:08,253 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/3d1465fe134e48eaa56d49921d3e67a3 is 50, key is test_row_0/C:col10/1731332226501/Put/seqid=0 2024-11-11T13:37:08,260 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/43e1c22bc78d467ca0ab090520cf97bc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/43e1c22bc78d467ca0ab090520cf97bc 2024-11-11T13:37:08,280 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into 43e1c22bc78d467ca0ab090520cf97bc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:08,280 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:08,280 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=13, startTime=1731332227760; duration=0sec 2024-11-11T13:37:08,280 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:08,280 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:08,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742261_1437 (size=12104) 2024-11-11T13:37:08,296 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/3d1465fe134e48eaa56d49921d3e67a3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/3d1465fe134e48eaa56d49921d3e67a3 2024-11-11T13:37:08,301 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into 3d1465fe134e48eaa56d49921d3e67a3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:08,301 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:08,301 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=13, startTime=1731332227760; duration=0sec 2024-11-11T13:37:08,301 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:08,301 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:08,332 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/e048c3329bd546c08234f8d769781238 2024-11-11T13:37:08,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/2d6b55b8c5e84d2ab51957095a96c5c4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2d6b55b8c5e84d2ab51957095a96c5c4 2024-11-11T13:37:08,342 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2d6b55b8c5e84d2ab51957095a96c5c4, entries=150, sequenceid=74, filesize=11.7 K 2024-11-11T13:37:08,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/71d13b719ce34acca8f194134517934c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/71d13b719ce34acca8f194134517934c 2024-11-11T13:37:08,348 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/71d13b719ce34acca8f194134517934c, entries=150, sequenceid=74, filesize=11.7 K 2024-11-11T13:37:08,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/e048c3329bd546c08234f8d769781238 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/e048c3329bd546c08234f8d769781238 2024-11-11T13:37:08,359 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/e048c3329bd546c08234f8d769781238, entries=150, sequenceid=74, filesize=11.7 K 2024-11-11T13:37:08,360 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for ed96da846868b1163b5c13dff6f86179 in 477ms, sequenceid=74, compaction requested=false 2024-11-11T13:37:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:08,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-11T13:37:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-11T13:37:08,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T13:37:08,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5490 sec 2024-11-11T13:37:08,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.5540 sec 2024-11-11T13:37:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:08,708 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:08,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:37:08,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/2b8f8d39b24642f9856668676a884d41 is 50, key is test_row_0/A:col10/1731332228701/Put/seqid=0 2024-11-11T13:37:08,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742262_1438 (size=14341) 2024-11-11T13:37:08,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/2b8f8d39b24642f9856668676a884d41 2024-11-11T13:37:08,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/fb2ccf6d3cbb482dad3b6b0f96fc032a is 50, key is test_row_0/B:col10/1731332228701/Put/seqid=0 2024-11-11T13:37:08,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742263_1439 (size=12001) 2024-11-11T13:37:08,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/fb2ccf6d3cbb482dad3b6b0f96fc032a 2024-11-11T13:37:08,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/581824db58c7422890498cabf7d848a0 is 50, key is test_row_0/C:col10/1731332228701/Put/seqid=0 2024-11-11T13:37:08,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742264_1440 (size=12001) 2024-11-11T13:37:08,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332288750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:08,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/581824db58c7422890498cabf7d848a0 2024-11-11T13:37:08,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/2b8f8d39b24642f9856668676a884d41 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2b8f8d39b24642f9856668676a884d41 2024-11-11T13:37:08,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332288751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:08,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:08,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332288753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:08,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332288755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:08,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:08,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332288756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:08,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2b8f8d39b24642f9856668676a884d41, entries=200, sequenceid=88, filesize=14.0 K 2024-11-11T13:37:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:08,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/fb2ccf6d3cbb482dad3b6b0f96fc032a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/fb2ccf6d3cbb482dad3b6b0f96fc032a 2024-11-11T13:37:08,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/fb2ccf6d3cbb482dad3b6b0f96fc032a, entries=150, sequenceid=88, filesize=11.7 K 2024-11-11T13:37:08,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/581824db58c7422890498cabf7d848a0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/581824db58c7422890498cabf7d848a0 2024-11-11T13:37:08,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/581824db58c7422890498cabf7d848a0, entries=150, sequenceid=88, filesize=11.7 K 2024-11-11T13:37:08,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ed96da846868b1163b5c13dff6f86179 in 74ms, sequenceid=88, compaction requested=true 2024-11-11T13:37:08,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:08,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:08,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:08,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:08,782 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:08,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:08,782 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:08,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:08,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:08,783 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:08,783 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:08,783 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:08,783 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:08,783 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:08,783 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:08,783 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8f24b5023eb7409bab408ad0928a61bd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/71d13b719ce34acca8f194134517934c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/fb2ccf6d3cbb482dad3b6b0f96fc032a] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=35.3 K 2024-11-11T13:37:08,783 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/43e1c22bc78d467ca0ab090520cf97bc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2d6b55b8c5e84d2ab51957095a96c5c4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2b8f8d39b24642f9856668676a884d41] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=37.5 K 2024-11-11T13:37:08,783 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f24b5023eb7409bab408ad0928a61bd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731332225870 2024-11-11T13:37:08,783 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e1c22bc78d467ca0ab090520cf97bc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731332225870 2024-11-11T13:37:08,784 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 71d13b719ce34acca8f194134517934c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731332226531 2024-11-11T13:37:08,784 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6b55b8c5e84d2ab51957095a96c5c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731332226531 2024-11-11T13:37:08,784 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting fb2ccf6d3cbb482dad3b6b0f96fc032a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731332228701 2024-11-11T13:37:08,784 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b8f8d39b24642f9856668676a884d41, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731332228701 2024-11-11T13:37:08,801 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:08,802 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/cb041dba78ff45a684218dde553e55c0 is 50, key is test_row_0/B:col10/1731332228701/Put/seqid=0 2024-11-11T13:37:08,805 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:08,805 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/5f8dbf2de9a34ef691a5cecfd6bce783 is 50, key is test_row_0/A:col10/1731332228701/Put/seqid=0 2024-11-11T13:37:08,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742265_1441 (size=12207) 2024-11-11T13:37:08,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742266_1442 (size=12207) 2024-11-11T13:37:08,830 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/cb041dba78ff45a684218dde553e55c0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/cb041dba78ff45a684218dde553e55c0 2024-11-11T13:37:08,846 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into cb041dba78ff45a684218dde553e55c0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:08,846 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:08,846 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=13, startTime=1731332228782; duration=0sec 2024-11-11T13:37:08,847 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:08,847 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:08,847 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:08,848 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/5f8dbf2de9a34ef691a5cecfd6bce783 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5f8dbf2de9a34ef691a5cecfd6bce783 2024-11-11T13:37:08,848 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:08,848 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:08,848 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:08,848 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/3d1465fe134e48eaa56d49921d3e67a3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/e048c3329bd546c08234f8d769781238, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/581824db58c7422890498cabf7d848a0] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=35.3 K 2024-11-11T13:37:08,849 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d1465fe134e48eaa56d49921d3e67a3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731332225870 2024-11-11T13:37:08,850 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e048c3329bd546c08234f8d769781238, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731332226531 2024-11-11T13:37:08,852 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 581824db58c7422890498cabf7d848a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731332228701 2024-11-11T13:37:08,852 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into 5f8dbf2de9a34ef691a5cecfd6bce783(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:08,852 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:08,852 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=13, startTime=1731332228782; duration=0sec 2024-11-11T13:37:08,852 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:08,852 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:08,865 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:08,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T13:37:08,865 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:08,866 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/8e5e3ee7c64c4cd2ba75ba1a0db92644 is 50, key is test_row_0/C:col10/1731332228701/Put/seqid=0 2024-11-11T13:37:08,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/a54a22cf799142d9a9b962a6f6434361 is 50, key is test_row_0/A:col10/1731332228754/Put/seqid=0 2024-11-11T13:37:08,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332288880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:08,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:08,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332288881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:08,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:08,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332288891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:08,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:08,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332288893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:08,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332288896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:08,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742267_1443 (size=12207) 2024-11-11T13:37:08,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742268_1444 (size=14341) 2024-11-11T13:37:08,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/a54a22cf799142d9a9b962a6f6434361 2024-11-11T13:37:08,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/8b7107b267914dc0bf460d0322c38135 is 50, key is test_row_0/B:col10/1731332228754/Put/seqid=0 2024-11-11T13:37:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T13:37:08,959 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:08,960 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-11T13:37:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T13:37:08,962 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:08,962 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:08,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:08,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742269_1445 (size=12001) 2024-11-11T13:37:08,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/8b7107b267914dc0bf460d0322c38135 2024-11-11T13:37:08,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/4d191381509746659a21d466d92777c7 is 50, key is test_row_0/C:col10/1731332228754/Put/seqid=0 2024-11-11T13:37:08,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742270_1446 (size=12001) 2024-11-11T13:37:09,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332288999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332288999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332289009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332289010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332289019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T13:37:09,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T13:37:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:09,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:09,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:09,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332289210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332289211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332289220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332289229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332289240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T13:37:09,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:09,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:09,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:09,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T13:37:09,330 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/8e5e3ee7c64c4cd2ba75ba1a0db92644 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/8e5e3ee7c64c4cd2ba75ba1a0db92644 2024-11-11T13:37:09,344 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into 8e5e3ee7c64c4cd2ba75ba1a0db92644(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:09,344 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:09,344 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=13, startTime=1731332228782; duration=0sec 2024-11-11T13:37:09,345 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:09,345 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:09,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/4d191381509746659a21d466d92777c7 2024-11-11T13:37:09,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/a54a22cf799142d9a9b962a6f6434361 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a54a22cf799142d9a9b962a6f6434361 2024-11-11T13:37:09,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a54a22cf799142d9a9b962a6f6434361, entries=200, sequenceid=116, filesize=14.0 K 2024-11-11T13:37:09,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/8b7107b267914dc0bf460d0322c38135 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8b7107b267914dc0bf460d0322c38135 2024-11-11T13:37:09,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8b7107b267914dc0bf460d0322c38135, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T13:37:09,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/4d191381509746659a21d466d92777c7 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4d191381509746659a21d466d92777c7 2024-11-11T13:37:09,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4d191381509746659a21d466d92777c7, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T13:37:09,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ed96da846868b1163b5c13dff6f86179 in 569ms, sequenceid=116, compaction requested=false 2024-11-11T13:37:09,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:09,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T13:37:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:09,443 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-11T13:37:09,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/00f050ae46124af5a17bab7ee5fb93e4 is 50, key is test_row_0/A:col10/1731332228891/Put/seqid=0 2024-11-11T13:37:09,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742271_1447 (size=12001) 2024-11-11T13:37:09,460 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/00f050ae46124af5a17bab7ee5fb93e4 2024-11-11T13:37:09,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/52d2d8ab50644f13aaf3eea4581933d4 is 50, key is test_row_0/B:col10/1731332228891/Put/seqid=0 2024-11-11T13:37:09,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742272_1448 (size=12001) 2024-11-11T13:37:09,492 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/52d2d8ab50644f13aaf3eea4581933d4 2024-11-11T13:37:09,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/24983221302e42a4b0286687a4755a33 is 50, key is test_row_0/C:col10/1731332228891/Put/seqid=0 2024-11-11T13:37:09,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742273_1449 (size=12001) 2024-11-11T13:37:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:09,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T13:37:09,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332289589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332289590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332289596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332289597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332289598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332289709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332289709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332289709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332289709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332289710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,906 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/24983221302e42a4b0286687a4755a33 2024-11-11T13:37:09,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/00f050ae46124af5a17bab7ee5fb93e4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/00f050ae46124af5a17bab7ee5fb93e4 2024-11-11T13:37:09,916 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/00f050ae46124af5a17bab7ee5fb93e4, entries=150, sequenceid=127, filesize=11.7 K 2024-11-11T13:37:09,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/52d2d8ab50644f13aaf3eea4581933d4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/52d2d8ab50644f13aaf3eea4581933d4 2024-11-11T13:37:09,921 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/52d2d8ab50644f13aaf3eea4581933d4, entries=150, sequenceid=127, filesize=11.7 K 2024-11-11T13:37:09,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/24983221302e42a4b0286687a4755a33 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/24983221302e42a4b0286687a4755a33 2024-11-11T13:37:09,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332289920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,925 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/24983221302e42a4b0286687a4755a33, entries=150, sequenceid=127, filesize=11.7 K 2024-11-11T13:37:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,926 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for ed96da846868b1163b5c13dff6f86179 in 483ms, sequenceid=127, compaction requested=true 2024-11-11T13:37:09,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:09,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:09,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-11T13:37:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-11T13:37:09,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-11T13:37:09,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 967 msec 2024-11-11T13:37:09,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 973 msec 2024-11-11T13:37:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:09,936 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:09,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-11T13:37:09,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/1ce568dc4ac741e1b49f33481cc69232 is 50, key is test_row_0/A:col10/1731332229596/Put/seqid=0 2024-11-11T13:37:09,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332289951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332289959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332289960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332289961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:09,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742274_1450 (size=12151) 2024-11-11T13:37:09,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/1ce568dc4ac741e1b49f33481cc69232 2024-11-11T13:37:09,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/f288f299bf1b4050a3a1d445437cc617 is 50, key is test_row_0/B:col10/1731332229596/Put/seqid=0 2024-11-11T13:37:10,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742275_1451 (size=12151) 2024-11-11T13:37:10,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/f288f299bf1b4050a3a1d445437cc617 2024-11-11T13:37:10,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/bc87bd7701224b36b3c8e80f8c4b036f is 50, key is test_row_0/C:col10/1731332229596/Put/seqid=0 2024-11-11T13:37:10,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742276_1452 (size=12151) 2024-11-11T13:37:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332290070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332290079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332290079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332290089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T13:37:10,101 INFO [Async-Client-Retry-Timer-pool-0 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:10,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-11T13:37:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T13:37:10,106 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:10,107 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:10,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T13:37:10,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332290229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T13:37:10,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:10,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:10,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:10,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:10,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:10,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332290279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332290289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332290289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332290312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T13:37:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:10,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:10,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T13:37:10,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/bc87bd7701224b36b3c8e80f8c4b036f 2024-11-11T13:37:10,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/1ce568dc4ac741e1b49f33481cc69232 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1ce568dc4ac741e1b49f33481cc69232 2024-11-11T13:37:10,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1ce568dc4ac741e1b49f33481cc69232, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T13:37:10,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/f288f299bf1b4050a3a1d445437cc617 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/f288f299bf1b4050a3a1d445437cc617 2024-11-11T13:37:10,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/f288f299bf1b4050a3a1d445437cc617, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T13:37:10,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/bc87bd7701224b36b3c8e80f8c4b036f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/bc87bd7701224b36b3c8e80f8c4b036f 2024-11-11T13:37:10,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/bc87bd7701224b36b3c8e80f8c4b036f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T13:37:10,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for ed96da846868b1163b5c13dff6f86179 in 609ms, sequenceid=155, compaction requested=true 2024-11-11T13:37:10,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:10,545 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:10,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:10,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:10,545 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:10,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:10,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:10,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:10,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:10,549 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:10,550 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:10,550 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:10,550 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5f8dbf2de9a34ef691a5cecfd6bce783, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a54a22cf799142d9a9b962a6f6434361, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/00f050ae46124af5a17bab7ee5fb93e4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1ce568dc4ac741e1b49f33481cc69232] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=49.5 K 2024-11-11T13:37:10,552 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:10,552 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:10,552 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:10,552 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/cb041dba78ff45a684218dde553e55c0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8b7107b267914dc0bf460d0322c38135, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/52d2d8ab50644f13aaf3eea4581933d4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/f288f299bf1b4050a3a1d445437cc617] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=47.2 K 2024-11-11T13:37:10,552 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f8dbf2de9a34ef691a5cecfd6bce783, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731332228701 2024-11-11T13:37:10,552 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting cb041dba78ff45a684218dde553e55c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731332228701 2024-11-11T13:37:10,552 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a54a22cf799142d9a9b962a6f6434361, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731332228750 2024-11-11T13:37:10,552 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b7107b267914dc0bf460d0322c38135, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731332228753 2024-11-11T13:37:10,553 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00f050ae46124af5a17bab7ee5fb93e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731332228879 2024-11-11T13:37:10,555 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce568dc4ac741e1b49f33481cc69232, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731332229596 2024-11-11T13:37:10,555 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 52d2d8ab50644f13aaf3eea4581933d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731332228879 2024-11-11T13:37:10,556 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f288f299bf1b4050a3a1d445437cc617, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731332229596 2024-11-11T13:37:10,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T13:37:10,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:10,574 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-11T13:37:10,580 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/156c5483c2a4420d911157895a89b129 is 50, key is test_row_0/A:col10/1731332229942/Put/seqid=0 2024-11-11T13:37:10,583 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:10,583 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/c6b0fa434eb44197affcb58787777b92 is 50, key is test_row_0/B:col10/1731332229596/Put/seqid=0 2024-11-11T13:37:10,584 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/33a54e7034294175909cabe02a59043d is 50, key is test_row_0/A:col10/1731332229596/Put/seqid=0 2024-11-11T13:37:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:10,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:10,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742277_1453 (size=9757) 2024-11-11T13:37:10,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742278_1454 (size=12493) 2024-11-11T13:37:10,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742279_1455 (size=12493) 2024-11-11T13:37:10,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332290658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332290670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332290671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332290672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T13:37:10,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332290739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332290779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332290779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332290779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332290789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332290989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332290989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332290989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,009 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/156c5483c2a4420d911157895a89b129 2024-11-11T13:37:11,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332291000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,021 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/c6b0fa434eb44197affcb58787777b92 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/c6b0fa434eb44197affcb58787777b92 2024-11-11T13:37:11,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/1a4907ad613f45c495d343b25bdafbf2 is 50, key is test_row_0/B:col10/1731332229942/Put/seqid=0 2024-11-11T13:37:11,025 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/33a54e7034294175909cabe02a59043d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/33a54e7034294175909cabe02a59043d 2024-11-11T13:37:11,025 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into c6b0fa434eb44197affcb58787777b92(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:11,025 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:11,025 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=12, startTime=1731332230545; duration=0sec 2024-11-11T13:37:11,025 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:11,025 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:11,025 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:11,027 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:11,027 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:11,027 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,027 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/8e5e3ee7c64c4cd2ba75ba1a0db92644, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4d191381509746659a21d466d92777c7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/24983221302e42a4b0286687a4755a33, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/bc87bd7701224b36b3c8e80f8c4b036f] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=47.2 K 2024-11-11T13:37:11,039 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e5e3ee7c64c4cd2ba75ba1a0db92644, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731332228701 2024-11-11T13:37:11,043 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d191381509746659a21d466d92777c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731332228753 2024-11-11T13:37:11,046 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 24983221302e42a4b0286687a4755a33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731332228879 2024-11-11T13:37:11,046 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting bc87bd7701224b36b3c8e80f8c4b036f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731332229596 2024-11-11T13:37:11,056 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into 33a54e7034294175909cabe02a59043d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:11,056 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:11,056 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=12, startTime=1731332230545; duration=0sec 2024-11-11T13:37:11,056 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:11,056 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:11,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742280_1456 (size=9757) 2024-11-11T13:37:11,076 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/1a4907ad613f45c495d343b25bdafbf2 2024-11-11T13:37:11,086 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:11,089 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/9275fe30ebab4a4f8bca968f1999145c is 50, key is test_row_0/C:col10/1731332229596/Put/seqid=0 2024-11-11T13:37:11,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/2616e717d574488e833867dccde7df0f is 50, key is test_row_0/C:col10/1731332229942/Put/seqid=0 2024-11-11T13:37:11,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742282_1458 (size=9757) 2024-11-11T13:37:11,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742281_1457 (size=12493) 2024-11-11T13:37:11,127 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/2616e717d574488e833867dccde7df0f 2024-11-11T13:37:11,135 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/9275fe30ebab4a4f8bca968f1999145c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/9275fe30ebab4a4f8bca968f1999145c 2024-11-11T13:37:11,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/156c5483c2a4420d911157895a89b129 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/156c5483c2a4420d911157895a89b129 2024-11-11T13:37:11,138 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/156c5483c2a4420d911157895a89b129, entries=100, sequenceid=164, filesize=9.5 K 2024-11-11T13:37:11,139 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into 9275fe30ebab4a4f8bca968f1999145c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:11,139 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:11,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/1a4907ad613f45c495d343b25bdafbf2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a4907ad613f45c495d343b25bdafbf2 2024-11-11T13:37:11,139 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=12, startTime=1731332230546; duration=0sec 2024-11-11T13:37:11,140 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:11,140 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:11,142 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a4907ad613f45c495d343b25bdafbf2, entries=100, sequenceid=164, filesize=9.5 K 2024-11-11T13:37:11,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/2616e717d574488e833867dccde7df0f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2616e717d574488e833867dccde7df0f 2024-11-11T13:37:11,148 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2616e717d574488e833867dccde7df0f, entries=100, sequenceid=164, filesize=9.5 K 2024-11-11T13:37:11,148 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for ed96da846868b1163b5c13dff6f86179 in 574ms, sequenceid=164, compaction requested=false 2024-11-11T13:37:11,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:11,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-11T13:37:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-11T13:37:11,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-11T13:37:11,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0430 sec 2024-11-11T13:37:11,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.0500 sec 2024-11-11T13:37:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T13:37:11,239 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:11,239 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-11T13:37:11,241 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T13:37:11,241 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:11,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:11,314 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:11,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-11T13:37:11,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/ea9f7baa592f455781351c0c733e10ca is 50, key is test_row_0/A:col10/1731332230669/Put/seqid=0 2024-11-11T13:37:11,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332291317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332291319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332291321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742283_1459 (size=14541) 2024-11-11T13:37:11,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332291325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/ea9f7baa592f455781351c0c733e10ca 2024-11-11T13:37:11,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/1a1ddd1b432542869836ecf2405c1899 is 50, key is test_row_0/B:col10/1731332230669/Put/seqid=0 2024-11-11T13:37:11,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742284_1460 (size=12151) 2024-11-11T13:37:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T13:37:11,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T13:37:11,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:11,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332291431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332291431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332291453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T13:37:11,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:11,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T13:37:11,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332291640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332291640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332291669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T13:37:11,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:11,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/1a1ddd1b432542869836ecf2405c1899 2024-11-11T13:37:11,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/2dd3a1e76fa14a45bf433a1e50e8bc6c is 50, key is test_row_0/C:col10/1731332230669/Put/seqid=0 2024-11-11T13:37:11,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742285_1461 (size=12151) 2024-11-11T13:37:11,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/2dd3a1e76fa14a45bf433a1e50e8bc6c 2024-11-11T13:37:11,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/ea9f7baa592f455781351c0c733e10ca as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/ea9f7baa592f455781351c0c733e10ca 2024-11-11T13:37:11,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332291759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,766 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in ed96da846868b1163b5c13dff6f86179 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:11,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:11,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/ea9f7baa592f455781351c0c733e10ca, entries=200, sequenceid=196, filesize=14.2 K 2024-11-11T13:37:11,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/1a1ddd1b432542869836ecf2405c1899 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a1ddd1b432542869836ecf2405c1899 2024-11-11T13:37:11,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a1ddd1b432542869836ecf2405c1899, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T13:37:11,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/2dd3a1e76fa14a45bf433a1e50e8bc6c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2dd3a1e76fa14a45bf433a1e50e8bc6c 2024-11-11T13:37:11,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2dd3a1e76fa14a45bf433a1e50e8bc6c, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T13:37:11,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for ed96da846868b1163b5c13dff6f86179 in 477ms, sequenceid=196, compaction requested=true 2024-11-11T13:37:11,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:11,791 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:11,792 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:11,794 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:11,794 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:11,794 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:11,794 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:11,794 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,794 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,794 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/c6b0fa434eb44197affcb58787777b92, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a4907ad613f45c495d343b25bdafbf2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a1ddd1b432542869836ecf2405c1899] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=33.6 K 2024-11-11T13:37:11,794 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/33a54e7034294175909cabe02a59043d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/156c5483c2a4420d911157895a89b129, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/ea9f7baa592f455781351c0c733e10ca] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=35.9 K 2024-11-11T13:37:11,794 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33a54e7034294175909cabe02a59043d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731332229596 2024-11-11T13:37:11,795 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c6b0fa434eb44197affcb58787777b92, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731332229596 2024-11-11T13:37:11,795 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 156c5483c2a4420d911157895a89b129, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731332229942 2024-11-11T13:37:11,795 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a4907ad613f45c495d343b25bdafbf2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731332229942 2024-11-11T13:37:11,795 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea9f7baa592f455781351c0c733e10ca, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332230668 2024-11-11T13:37:11,795 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a1ddd1b432542869836ecf2405c1899, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332230669 2024-11-11T13:37:11,807 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#390 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:11,807 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/f003b99ac9d642b8af232c3d756c04cd is 50, key is test_row_0/B:col10/1731332230669/Put/seqid=0 2024-11-11T13:37:11,809 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:11,810 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/02bff4b5564f467c83db7762bd22515f is 50, key is test_row_0/A:col10/1731332230669/Put/seqid=0 2024-11-11T13:37:11,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742286_1462 (size=12595) 2024-11-11T13:37:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:11,844 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:11,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:37:11,846 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/f003b99ac9d642b8af232c3d756c04cd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/f003b99ac9d642b8af232c3d756c04cd 2024-11-11T13:37:11,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742287_1463 (size=12595) 2024-11-11T13:37:11,855 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into f003b99ac9d642b8af232c3d756c04cd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:11,855 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:11,855 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=13, startTime=1731332231792; duration=0sec 2024-11-11T13:37:11,856 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:11,856 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:11,856 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:11,858 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:11,858 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:11,858 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,858 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/9275fe30ebab4a4f8bca968f1999145c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2616e717d574488e833867dccde7df0f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2dd3a1e76fa14a45bf433a1e50e8bc6c] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=33.6 K 2024-11-11T13:37:11,859 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 9275fe30ebab4a4f8bca968f1999145c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731332229596 2024-11-11T13:37:11,859 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2616e717d574488e833867dccde7df0f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731332229942 2024-11-11T13:37:11,859 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dd3a1e76fa14a45bf433a1e50e8bc6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332230669 2024-11-11T13:37:11,862 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/02bff4b5564f467c83db7762bd22515f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/02bff4b5564f467c83db7762bd22515f 2024-11-11T13:37:11,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T13:37:11,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:11,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:11,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,867 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into 02bff4b5564f467c83db7762bd22515f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:11,867 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:11,867 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=13, startTime=1731332231791; duration=0sec 2024-11-11T13:37:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:11,867 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:11,867 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T13:37:11,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/5bb24b160a064955a8b616a38edf3a1a is 50, key is test_row_0/A:col10/1731332231837/Put/seqid=0 2024-11-11T13:37:11,888 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:11,888 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/0a6665b744f84a799327882bf110a07d is 50, key is test_row_0/C:col10/1731332230669/Put/seqid=0 2024-11-11T13:37:11,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742288_1464 (size=14541) 2024-11-11T13:37:11,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742289_1465 (size=12595) 2024-11-11T13:37:11,907 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/0a6665b744f84a799327882bf110a07d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/0a6665b744f84a799327882bf110a07d 2024-11-11T13:37:11,916 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into 0a6665b744f84a799327882bf110a07d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:11,916 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:11,916 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=13, startTime=1731332231792; duration=0sec 2024-11-11T13:37:11,916 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:11,916 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:11,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332291983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332291988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332291990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332291992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T13:37:12,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:12,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:12,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:12,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332292109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332292109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332292109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T13:37:12,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:12,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:12,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:12,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/5bb24b160a064955a8b616a38edf3a1a 2024-11-11T13:37:12,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/e9b5f88199864a7f88566af05546574a is 50, key is test_row_0/B:col10/1731332231837/Put/seqid=0 2024-11-11T13:37:12,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742290_1466 (size=12151) 2024-11-11T13:37:12,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T13:37:12,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:12,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:12,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:12,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332292329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332292329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332292329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T13:37:12,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T13:37:12,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:12,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332292503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T13:37:12,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:12,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:12,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:12,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332292643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332292651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332292656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:12,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/e9b5f88199864a7f88566af05546574a 2024-11-11T13:37:12,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/db4dc401f46f41d3ab9e00331514346f is 50, key is test_row_0/C:col10/1731332231837/Put/seqid=0 2024-11-11T13:37:12,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742291_1467 (size=12151) 2024-11-11T13:37:12,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/db4dc401f46f41d3ab9e00331514346f 2024-11-11T13:37:12,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/5bb24b160a064955a8b616a38edf3a1a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5bb24b160a064955a8b616a38edf3a1a 2024-11-11T13:37:12,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5bb24b160a064955a8b616a38edf3a1a, entries=200, sequenceid=207, filesize=14.2 K 2024-11-11T13:37:12,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/e9b5f88199864a7f88566af05546574a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/e9b5f88199864a7f88566af05546574a 2024-11-11T13:37:12,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/e9b5f88199864a7f88566af05546574a, entries=150, sequenceid=207, filesize=11.9 K 2024-11-11T13:37:12,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/db4dc401f46f41d3ab9e00331514346f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/db4dc401f46f41d3ab9e00331514346f 2024-11-11T13:37:12,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/db4dc401f46f41d3ab9e00331514346f, entries=150, sequenceid=207, filesize=11.9 K 2024-11-11T13:37:12,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for ed96da846868b1163b5c13dff6f86179 in 902ms, sequenceid=207, compaction requested=false 2024-11-11T13:37:12,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:12,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T13:37:12,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:12,810 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T13:37:12,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/b75bfa54e40d4ab09d3d8d6aece0d04e is 50, key is test_row_0/A:col10/1731332231984/Put/seqid=0 2024-11-11T13:37:12,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742292_1468 (size=12151) 2024-11-11T13:37:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:13,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:13,218 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/b75bfa54e40d4ab09d3d8d6aece0d04e 2024-11-11T13:37:13,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332293219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332293219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332293219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/b35de2ea6200413da0299e2ba6cd3cd6 is 50, key is test_row_0/B:col10/1731332231984/Put/seqid=0 2024-11-11T13:37:13,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742293_1469 (size=12151) 2024-11-11T13:37:13,280 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/b35de2ea6200413da0299e2ba6cd3cd6 2024-11-11T13:37:13,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/6d61a344242b42608fd4e7227c3ef6d7 is 50, key is test_row_0/C:col10/1731332231984/Put/seqid=0 2024-11-11T13:37:13,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332293339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332293340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332293340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742294_1470 (size=12151) 2024-11-11T13:37:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T13:37:13,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332293529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,537 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in ed96da846868b1163b5c13dff6f86179 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332293550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332293550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332293550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,747 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/6d61a344242b42608fd4e7227c3ef6d7 2024-11-11T13:37:13,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/b75bfa54e40d4ab09d3d8d6aece0d04e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b75bfa54e40d4ab09d3d8d6aece0d04e 2024-11-11T13:37:13,767 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b75bfa54e40d4ab09d3d8d6aece0d04e, entries=150, sequenceid=236, filesize=11.9 K 2024-11-11T13:37:13,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/b35de2ea6200413da0299e2ba6cd3cd6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b35de2ea6200413da0299e2ba6cd3cd6 2024-11-11T13:37:13,780 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b35de2ea6200413da0299e2ba6cd3cd6, entries=150, sequenceid=236, filesize=11.9 K 2024-11-11T13:37:13,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/6d61a344242b42608fd4e7227c3ef6d7 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/6d61a344242b42608fd4e7227c3ef6d7 2024-11-11T13:37:13,787 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/6d61a344242b42608fd4e7227c3ef6d7, entries=150, sequenceid=236, filesize=11.9 K 2024-11-11T13:37:13,788 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for ed96da846868b1163b5c13dff6f86179 in 978ms, sequenceid=236, compaction requested=true 2024-11-11T13:37:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:13,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-11T13:37:13,790 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:13,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:37:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:13,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-11T13:37:13,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T13:37:13,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5520 sec 2024-11-11T13:37:13,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.5560 sec 2024-11-11T13:37:13,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/d7d060c50dbf49de938a593b4233830d is 50, key is test_row_0/A:col10/1731332233188/Put/seqid=0 2024-11-11T13:37:13,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742295_1471 (size=14541) 2024-11-11T13:37:13,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/d7d060c50dbf49de938a593b4233830d 2024-11-11T13:37:13,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/d37b5aa3416046f69b6b768dbc0f90b2 is 50, key is test_row_0/B:col10/1731332233188/Put/seqid=0 2024-11-11T13:37:13,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742296_1472 (size=12151) 2024-11-11T13:37:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332293892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332293896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332293897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:13,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332293902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:13,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:13,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332294009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332294010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332294019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332294019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332294219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332294229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332294229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332294229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/d37b5aa3416046f69b6b768dbc0f90b2 2024-11-11T13:37:14,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/0b1b2b11eff34af49ee8dfe0235afcf1 is 50, key is test_row_0/C:col10/1731332233188/Put/seqid=0 2024-11-11T13:37:14,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742297_1473 (size=12151) 2024-11-11T13:37:14,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332294530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332294539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332294555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332294556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:14,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:14,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/0b1b2b11eff34af49ee8dfe0235afcf1 2024-11-11T13:37:14,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/d7d060c50dbf49de938a593b4233830d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/d7d060c50dbf49de938a593b4233830d 2024-11-11T13:37:14,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/d7d060c50dbf49de938a593b4233830d, entries=200, sequenceid=247, filesize=14.2 K 2024-11-11T13:37:14,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/d37b5aa3416046f69b6b768dbc0f90b2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/d37b5aa3416046f69b6b768dbc0f90b2 2024-11-11T13:37:14,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/d37b5aa3416046f69b6b768dbc0f90b2, entries=150, sequenceid=247, filesize=11.9 K 2024-11-11T13:37:14,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/0b1b2b11eff34af49ee8dfe0235afcf1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/0b1b2b11eff34af49ee8dfe0235afcf1 2024-11-11T13:37:14,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/0b1b2b11eff34af49ee8dfe0235afcf1, entries=150, sequenceid=247, filesize=11.9 K 2024-11-11T13:37:14,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ed96da846868b1163b5c13dff6f86179 in 976ms, sequenceid=247, compaction requested=true 2024-11-11T13:37:14,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:14,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:14,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:14,766 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:14,766 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:14,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:14,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:14,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:14,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:14,768 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:14,768 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:14,768 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:14,768 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/f003b99ac9d642b8af232c3d756c04cd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/e9b5f88199864a7f88566af05546574a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b35de2ea6200413da0299e2ba6cd3cd6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/d37b5aa3416046f69b6b768dbc0f90b2] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=47.9 K 2024-11-11T13:37:14,768 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:14,768 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:14,768 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:14,768 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/02bff4b5564f467c83db7762bd22515f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5bb24b160a064955a8b616a38edf3a1a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b75bfa54e40d4ab09d3d8d6aece0d04e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/d7d060c50dbf49de938a593b4233830d] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=52.6 K 2024-11-11T13:37:14,776 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f003b99ac9d642b8af232c3d756c04cd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332230669 2024-11-11T13:37:14,776 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02bff4b5564f467c83db7762bd22515f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332230669 2024-11-11T13:37:14,779 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bb24b160a064955a8b616a38edf3a1a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731332231324 2024-11-11T13:37:14,779 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b5f88199864a7f88566af05546574a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731332231324 2024-11-11T13:37:14,781 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting b35de2ea6200413da0299e2ba6cd3cd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731332231976 2024-11-11T13:37:14,781 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting b75bfa54e40d4ab09d3d8d6aece0d04e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731332231976 2024-11-11T13:37:14,787 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7d060c50dbf49de938a593b4233830d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731332233187 2024-11-11T13:37:14,787 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting d37b5aa3416046f69b6b768dbc0f90b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731332233188 2024-11-11T13:37:14,847 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:14,848 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/304536c32cf948aaaa314058d7290bfd is 50, key is test_row_0/A:col10/1731332233188/Put/seqid=0 2024-11-11T13:37:14,858 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:14,859 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/9a331355e17c4fbba84c2b828b787c37 is 50, key is test_row_0/B:col10/1731332233188/Put/seqid=0 2024-11-11T13:37:14,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742299_1475 (size=12731) 2024-11-11T13:37:14,916 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/9a331355e17c4fbba84c2b828b787c37 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/9a331355e17c4fbba84c2b828b787c37 2024-11-11T13:37:14,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742298_1474 (size=12731) 2024-11-11T13:37:14,924 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into 9a331355e17c4fbba84c2b828b787c37(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:14,924 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:14,924 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=12, startTime=1731332234766; duration=0sec 2024-11-11T13:37:14,924 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:14,924 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:14,924 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:14,929 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/304536c32cf948aaaa314058d7290bfd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/304536c32cf948aaaa314058d7290bfd 2024-11-11T13:37:14,933 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:14,933 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:14,933 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:14,933 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/0a6665b744f84a799327882bf110a07d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/db4dc401f46f41d3ab9e00331514346f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/6d61a344242b42608fd4e7227c3ef6d7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/0b1b2b11eff34af49ee8dfe0235afcf1] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=47.9 K 2024-11-11T13:37:14,934 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6665b744f84a799327882bf110a07d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332230669 2024-11-11T13:37:14,934 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting db4dc401f46f41d3ab9e00331514346f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731332231324 2024-11-11T13:37:14,934 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d61a344242b42608fd4e7227c3ef6d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731332231976 2024-11-11T13:37:14,935 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into 304536c32cf948aaaa314058d7290bfd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:14,935 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b1b2b11eff34af49ee8dfe0235afcf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731332233188 2024-11-11T13:37:14,935 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:14,935 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=12, startTime=1731332234766; duration=0sec 2024-11-11T13:37:14,935 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:14,935 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:14,942 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:14,943 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/b5e57f9370db4fcc8e53856b4b03aeaf is 50, key is test_row_0/C:col10/1731332233188/Put/seqid=0 2024-11-11T13:37:14,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742300_1476 (size=12731) 2024-11-11T13:37:14,983 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/b5e57f9370db4fcc8e53856b4b03aeaf as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/b5e57f9370db4fcc8e53856b4b03aeaf 2024-11-11T13:37:14,992 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into b5e57f9370db4fcc8e53856b4b03aeaf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:14,992 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:14,992 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=12, startTime=1731332234767; duration=0sec 2024-11-11T13:37:14,992 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:14,992 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:15,044 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:15,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-11T13:37:15,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/4747057b0a79423bbddf568fea03e93b is 50, key is test_row_0/A:col10/1731332235043/Put/seqid=0 2024-11-11T13:37:15,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742301_1477 (size=14741) 2024-11-11T13:37:15,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332295074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332295081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332295082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332295082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332295189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332295189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332295199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332295199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332295400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332295401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T13:37:15,409 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:15,410 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-11T13:37:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T13:37:15,413 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:15,414 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:15,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:15,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332295409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332295410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/4747057b0a79423bbddf568fea03e93b 2024-11-11T13:37:15,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/aa1e580967704524a12b1fd12a70c4c9 is 50, key is test_row_0/B:col10/1731332235043/Put/seqid=0 2024-11-11T13:37:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T13:37:15,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742302_1478 (size=12301) 2024-11-11T13:37:15,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/aa1e580967704524a12b1fd12a70c4c9 2024-11-11T13:37:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332295550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,554 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in ed96da846868b1163b5c13dff6f86179 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/4a42843ccb1a4444be8a01a1de67d330 is 50, key is test_row_0/C:col10/1731332235043/Put/seqid=0 2024-11-11T13:37:15,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T13:37:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:15,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:15,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:15,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742303_1479 (size=12301) 2024-11-11T13:37:15,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332295710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332295710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T13:37:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:15,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T13:37:15,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332295729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332295729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:15,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T13:37:15,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:15,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:15,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:15,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:15,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:16,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/4a42843ccb1a4444be8a01a1de67d330 2024-11-11T13:37:16,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T13:37:16,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:16,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:16,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:16,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:16,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T13:37:16,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/4747057b0a79423bbddf568fea03e93b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/4747057b0a79423bbddf568fea03e93b 2024-11-11T13:37:16,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/4747057b0a79423bbddf568fea03e93b, entries=200, sequenceid=276, filesize=14.4 K 2024-11-11T13:37:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/aa1e580967704524a12b1fd12a70c4c9 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/aa1e580967704524a12b1fd12a70c4c9 2024-11-11T13:37:16,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/aa1e580967704524a12b1fd12a70c4c9, entries=150, sequenceid=276, filesize=12.0 K 2024-11-11T13:37:16,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/4a42843ccb1a4444be8a01a1de67d330 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4a42843ccb1a4444be8a01a1de67d330 2024-11-11T13:37:16,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4a42843ccb1a4444be8a01a1de67d330, entries=150, sequenceid=276, filesize=12.0 K 2024-11-11T13:37:16,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for ed96da846868b1163b5c13dff6f86179 in 1126ms, sequenceid=276, compaction requested=false 2024-11-11T13:37:16,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:16,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T13:37:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:16,198 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:37:16,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/92ce4cfe9bd44093be71e2aa5b57fa86 is 50, key is test_row_0/A:col10/1731332235065/Put/seqid=0 2024-11-11T13:37:16,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742304_1480 (size=12301) 2024-11-11T13:37:16,206 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/92ce4cfe9bd44093be71e2aa5b57fa86 2024-11-11T13:37:16,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/47cfefca41344bb18f5694f42635bb7f is 50, key is test_row_0/B:col10/1731332235065/Put/seqid=0 2024-11-11T13:37:16,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742305_1481 (size=12301) 2024-11-11T13:37:16,217 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/47cfefca41344bb18f5694f42635bb7f 2024-11-11T13:37:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:16,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:16,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/4b8e5113dbfc4d06a10efa7eb0fcea38 is 50, key is test_row_0/C:col10/1731332235065/Put/seqid=0 2024-11-11T13:37:16,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742306_1482 (size=12301) 2024-11-11T13:37:16,233 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/4b8e5113dbfc4d06a10efa7eb0fcea38 2024-11-11T13:37:16,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/92ce4cfe9bd44093be71e2aa5b57fa86 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92ce4cfe9bd44093be71e2aa5b57fa86 2024-11-11T13:37:16,243 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92ce4cfe9bd44093be71e2aa5b57fa86, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T13:37:16,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/47cfefca41344bb18f5694f42635bb7f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/47cfefca41344bb18f5694f42635bb7f 2024-11-11T13:37:16,248 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/47cfefca41344bb18f5694f42635bb7f, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T13:37:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/4b8e5113dbfc4d06a10efa7eb0fcea38 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4b8e5113dbfc4d06a10efa7eb0fcea38 2024-11-11T13:37:16,252 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4b8e5113dbfc4d06a10efa7eb0fcea38, entries=150, sequenceid=287, filesize=12.0 K 2024-11-11T13:37:16,253 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=67.09 KB/68700 for ed96da846868b1163b5c13dff6f86179 in 55ms, sequenceid=287, compaction requested=true 2024-11-11T13:37:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:16,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-11T13:37:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-11T13:37:16,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T13:37:16,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 840 msec 2024-11-11T13:37:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:16,258 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:16,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T13:37:16,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 847 msec 2024-11-11T13:37:16,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/69b7b4433ff04a94b90713d5f17b27c9 is 50, key is test_row_0/A:col10/1731332236244/Put/seqid=0 2024-11-11T13:37:16,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742307_1483 (size=14741) 2024-11-11T13:37:16,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332296301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332296302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332296303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332296309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332296419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332296419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332296420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332296420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T13:37:16,549 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:16,549 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-11T13:37:16,552 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T13:37:16,552 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:16,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:16,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332296630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332296639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332296639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332296639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T13:37:16,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/69b7b4433ff04a94b90713d5f17b27c9 2024-11-11T13:37:16,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/dd112f2f0a6840a7854bc4fae219719a is 50, key is test_row_0/B:col10/1731332236244/Put/seqid=0 2024-11-11T13:37:16,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T13:37:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:16,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:16,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742308_1484 (size=12301) 2024-11-11T13:37:16,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/dd112f2f0a6840a7854bc4fae219719a 2024-11-11T13:37:16,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/aa78ce59c21b4796ba63ff06dc8751cd is 50, key is test_row_0/C:col10/1731332236244/Put/seqid=0 2024-11-11T13:37:16,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742309_1485 (size=12301) 2024-11-11T13:37:16,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/aa78ce59c21b4796ba63ff06dc8751cd 2024-11-11T13:37:16,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/69b7b4433ff04a94b90713d5f17b27c9 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/69b7b4433ff04a94b90713d5f17b27c9 2024-11-11T13:37:16,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/69b7b4433ff04a94b90713d5f17b27c9, entries=200, sequenceid=301, filesize=14.4 K 2024-11-11T13:37:16,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/dd112f2f0a6840a7854bc4fae219719a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/dd112f2f0a6840a7854bc4fae219719a 2024-11-11T13:37:16,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/dd112f2f0a6840a7854bc4fae219719a, entries=150, sequenceid=301, filesize=12.0 K 2024-11-11T13:37:16,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/aa78ce59c21b4796ba63ff06dc8751cd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/aa78ce59c21b4796ba63ff06dc8751cd 2024-11-11T13:37:16,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/aa78ce59c21b4796ba63ff06dc8751cd, entries=150, sequenceid=301, filesize=12.0 K 2024-11-11T13:37:16,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ed96da846868b1163b5c13dff6f86179 in 551ms, sequenceid=301, compaction requested=true 2024-11-11T13:37:16,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:16,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:16,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:16,809 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:16,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:16,809 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:16,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:16,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:16,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:16,811 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:16,811 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:16,811 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:16,811 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:16,811 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:16,811 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:16,811 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/9a331355e17c4fbba84c2b828b787c37, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/aa1e580967704524a12b1fd12a70c4c9, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/47cfefca41344bb18f5694f42635bb7f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/dd112f2f0a6840a7854bc4fae219719a] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=48.5 K 2024-11-11T13:37:16,811 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/304536c32cf948aaaa314058d7290bfd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/4747057b0a79423bbddf568fea03e93b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92ce4cfe9bd44093be71e2aa5b57fa86, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/69b7b4433ff04a94b90713d5f17b27c9] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=53.2 K 2024-11-11T13:37:16,811 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a331355e17c4fbba84c2b828b787c37, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731332233188 2024-11-11T13:37:16,811 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 304536c32cf948aaaa314058d7290bfd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731332233188 2024-11-11T13:37:16,811 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting aa1e580967704524a12b1fd12a70c4c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731332233891 2024-11-11T13:37:16,812 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4747057b0a79423bbddf568fea03e93b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731332233891 2024-11-11T13:37:16,812 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 47cfefca41344bb18f5694f42635bb7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731332235065 2024-11-11T13:37:16,812 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92ce4cfe9bd44093be71e2aa5b57fa86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731332235065 2024-11-11T13:37:16,812 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting dd112f2f0a6840a7854bc4fae219719a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731332236244 2024-11-11T13:37:16,812 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69b7b4433ff04a94b90713d5f17b27c9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731332236244 2024-11-11T13:37:16,841 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:16,842 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/0ef00d8928cc45928f36cdb24635391f is 50, key is test_row_0/A:col10/1731332236244/Put/seqid=0 2024-11-11T13:37:16,853 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:16,854 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/2cbc816b155741e2bc10e4dd394320b6 is 50, key is test_row_0/B:col10/1731332236244/Put/seqid=0 2024-11-11T13:37:16,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T13:37:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:16,859 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T13:37:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T13:37:16,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742310_1486 (size=13017) 2024-11-11T13:37:16,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/136e528a903d471c9bc681ec692a791f is 50, key is test_row_0/A:col10/1731332236308/Put/seqid=0 2024-11-11T13:37:16,881 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/0ef00d8928cc45928f36cdb24635391f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/0ef00d8928cc45928f36cdb24635391f 2024-11-11T13:37:16,886 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into 0ef00d8928cc45928f36cdb24635391f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:16,886 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:16,886 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=12, startTime=1731332236809; duration=0sec 2024-11-11T13:37:16,886 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:16,886 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:16,886 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:16,887 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:16,888 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:16,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742311_1487 (size=13017) 2024-11-11T13:37:16,888 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:16,888 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/b5e57f9370db4fcc8e53856b4b03aeaf, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4a42843ccb1a4444be8a01a1de67d330, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4b8e5113dbfc4d06a10efa7eb0fcea38, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/aa78ce59c21b4796ba63ff06dc8751cd] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=48.5 K 2024-11-11T13:37:16,889 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5e57f9370db4fcc8e53856b4b03aeaf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731332233188 2024-11-11T13:37:16,889 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a42843ccb1a4444be8a01a1de67d330, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731332233891 2024-11-11T13:37:16,889 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b8e5113dbfc4d06a10efa7eb0fcea38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731332235065 2024-11-11T13:37:16,889 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa78ce59c21b4796ba63ff06dc8751cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731332236244 2024-11-11T13:37:16,894 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/2cbc816b155741e2bc10e4dd394320b6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/2cbc816b155741e2bc10e4dd394320b6 2024-11-11T13:37:16,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742312_1488 (size=12301) 2024-11-11T13:37:16,910 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into 2cbc816b155741e2bc10e4dd394320b6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:16,910 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:16,911 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=12, startTime=1731332236809; duration=0sec 2024-11-11T13:37:16,911 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:16,911 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:16,914 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:16,914 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/48af5f55e7474f70884484ca31f07c13 is 50, key is test_row_0/C:col10/1731332236244/Put/seqid=0 2024-11-11T13:37:16,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742313_1489 (size=13017) 2024-11-11T13:37:16,934 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/48af5f55e7474f70884484ca31f07c13 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/48af5f55e7474f70884484ca31f07c13 2024-11-11T13:37:16,945 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into 48af5f55e7474f70884484ca31f07c13(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:16,945 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:16,945 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=12, startTime=1731332236809; duration=0sec 2024-11-11T13:37:16,945 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:16,945 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:16,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:16,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332296977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332296981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332296981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:16,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332296983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332297099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332297100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332297107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332297107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T13:37:17,296 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/136e528a903d471c9bc681ec692a791f 2024-11-11T13:37:17,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a2d544a87dac4b798ac2f6b8cb6e56ae is 50, key is test_row_0/B:col10/1731332236308/Put/seqid=0 2024-11-11T13:37:17,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332297310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742314_1490 (size=12301) 2024-11-11T13:37:17,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332297319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332297329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332297329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332297619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332297629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332297639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332297639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T13:37:17,721 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a2d544a87dac4b798ac2f6b8cb6e56ae 2024-11-11T13:37:17,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/158967b15f314f95a12b9596dac6f484 is 50, key is test_row_0/C:col10/1731332236308/Put/seqid=0 2024-11-11T13:37:17,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742315_1491 (size=12301) 2024-11-11T13:37:18,141 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/158967b15f314f95a12b9596dac6f484 2024-11-11T13:37:18,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:18,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332298139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/136e528a903d471c9bc681ec692a791f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/136e528a903d471c9bc681ec692a791f 2024-11-11T13:37:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:18,148 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/136e528a903d471c9bc681ec692a791f, entries=150, sequenceid=323, filesize=12.0 K 2024-11-11T13:37:18,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a2d544a87dac4b798ac2f6b8cb6e56ae as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a2d544a87dac4b798ac2f6b8cb6e56ae 2024-11-11T13:37:18,151 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a2d544a87dac4b798ac2f6b8cb6e56ae, entries=150, sequenceid=323, filesize=12.0 K 2024-11-11T13:37:18,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/158967b15f314f95a12b9596dac6f484 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/158967b15f314f95a12b9596dac6f484 2024-11-11T13:37:18,154 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/158967b15f314f95a12b9596dac6f484, entries=150, sequenceid=323, filesize=12.0 K 2024-11-11T13:37:18,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332298149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:18,155 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ed96da846868b1163b5c13dff6f86179 in 1296ms, sequenceid=323, compaction requested=false 2024-11-11T13:37:18,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:18,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-11T13:37:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-11T13:37:18,159 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:18,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T13:37:18,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:18,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T13:37:18,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6070 sec 2024-11-11T13:37:18,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.6150 sec 2024-11-11T13:37:18,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/92cb1f1828444a708a924a07ae8498bd is 50, key is test_row_0/A:col10/1731332236982/Put/seqid=0 2024-11-11T13:37:18,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742316_1492 (size=17181) 2024-11-11T13:37:18,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332298301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:18,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332298301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:18,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332298429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:18,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332298429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:18,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/92cb1f1828444a708a924a07ae8498bd 2024-11-11T13:37:18,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/8e48fe112296485395c6fd2d99154181 is 50, key is test_row_0/B:col10/1731332236982/Put/seqid=0 2024-11-11T13:37:18,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742317_1493 (size=12301) 2024-11-11T13:37:18,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332298649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:18,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332298649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T13:37:18,700 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:18,700 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-11T13:37:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T13:37:18,702 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:18,702 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:18,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T13:37:18,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T13:37:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:18,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:18,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:18,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332298969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:18,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332298971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:18,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:18,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T13:37:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T13:37:19,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/8e48fe112296485395c6fd2d99154181 2024-11-11T13:37:19,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/8b89a19a6ff044c4a30186e99a4ca4d0 is 50, key is test_row_0/C:col10/1731332236982/Put/seqid=0 2024-11-11T13:37:19,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742318_1494 (size=12301) 2024-11-11T13:37:19,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/8b89a19a6ff044c4a30186e99a4ca4d0 2024-11-11T13:37:19,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/92cb1f1828444a708a924a07ae8498bd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92cb1f1828444a708a924a07ae8498bd 2024-11-11T13:37:19,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92cb1f1828444a708a924a07ae8498bd, entries=250, sequenceid=342, filesize=16.8 K 2024-11-11T13:37:19,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/8e48fe112296485395c6fd2d99154181 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8e48fe112296485395c6fd2d99154181 2024-11-11T13:37:19,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8e48fe112296485395c6fd2d99154181, entries=150, sequenceid=342, filesize=12.0 K 2024-11-11T13:37:19,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/8b89a19a6ff044c4a30186e99a4ca4d0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/8b89a19a6ff044c4a30186e99a4ca4d0 2024-11-11T13:37:19,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/8b89a19a6ff044c4a30186e99a4ca4d0, entries=150, sequenceid=342, filesize=12.0 K 2024-11-11T13:37:19,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for ed96da846868b1163b5c13dff6f86179 in 957ms, sequenceid=342, compaction requested=true 2024-11-11T13:37:19,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:19,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:19,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:19,116 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:19,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:19,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:19,116 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:19,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:19,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:19,120 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:19,120 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:19,120 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:19,120 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,124 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:19,124 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,125 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/0ef00d8928cc45928f36cdb24635391f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/136e528a903d471c9bc681ec692a791f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92cb1f1828444a708a924a07ae8498bd] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=41.5 K 2024-11-11T13:37:19,125 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/2cbc816b155741e2bc10e4dd394320b6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a2d544a87dac4b798ac2f6b8cb6e56ae, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8e48fe112296485395c6fd2d99154181] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=36.7 K 2024-11-11T13:37:19,125 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ef00d8928cc45928f36cdb24635391f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731332236244 2024-11-11T13:37:19,125 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cbc816b155741e2bc10e4dd394320b6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731332236244 2024-11-11T13:37:19,125 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 136e528a903d471c9bc681ec692a791f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731332236297 2024-11-11T13:37:19,126 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting a2d544a87dac4b798ac2f6b8cb6e56ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731332236297 2024-11-11T13:37:19,126 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92cb1f1828444a708a924a07ae8498bd, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731332236957 2024-11-11T13:37:19,126 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e48fe112296485395c6fd2d99154181, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731332236982 2024-11-11T13:37:19,161 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:19,162 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/0285cc7eeb4f409586b924850abcbdf3 is 50, key is test_row_0/B:col10/1731332236982/Put/seqid=0 2024-11-11T13:37:19,164 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#424 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:19,164 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/92fc34e9d7714d5185de5e841e212bc1 is 50, key is test_row_0/A:col10/1731332236982/Put/seqid=0 2024-11-11T13:37:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:19,167 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:19,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T13:37:19,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T13:37:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:19,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/86d0018787c6447381ed4c03244c929f is 50, key is test_row_0/A:col10/1731332238297/Put/seqid=0 2024-11-11T13:37:19,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742319_1495 (size=13119) 2024-11-11T13:37:19,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742320_1496 (size=13119) 2024-11-11T13:37:19,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742321_1497 (size=14741) 2024-11-11T13:37:19,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/86d0018787c6447381ed4c03244c929f 2024-11-11T13:37:19,231 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/92fc34e9d7714d5185de5e841e212bc1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92fc34e9d7714d5185de5e841e212bc1 2024-11-11T13:37:19,237 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into 92fc34e9d7714d5185de5e841e212bc1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:19,237 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:19,237 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=13, startTime=1731332239116; duration=0sec 2024-11-11T13:37:19,237 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:19,237 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:19,237 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:19,238 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:19,238 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:19,238 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,238 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/48af5f55e7474f70884484ca31f07c13, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/158967b15f314f95a12b9596dac6f484, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/8b89a19a6ff044c4a30186e99a4ca4d0] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=36.7 K 2024-11-11T13:37:19,239 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48af5f55e7474f70884484ca31f07c13, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731332236244 2024-11-11T13:37:19,239 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 158967b15f314f95a12b9596dac6f484, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731332236297 2024-11-11T13:37:19,239 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b89a19a6ff044c4a30186e99a4ca4d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731332236982 2024-11-11T13:37:19,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/ed18a0debea04b20a29a8dc5e40a2f2a is 50, key is test_row_0/B:col10/1731332238297/Put/seqid=0 2024-11-11T13:37:19,252 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:19,252 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/f62d493aac044044aa22c898a1286662 is 50, key is test_row_0/C:col10/1731332236982/Put/seqid=0 2024-11-11T13:37:19,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742323_1499 (size=13119) 2024-11-11T13:37:19,266 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/f62d493aac044044aa22c898a1286662 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f62d493aac044044aa22c898a1286662 2024-11-11T13:37:19,271 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into f62d493aac044044aa22c898a1286662(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:19,271 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:19,271 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=13, startTime=1731332239116; duration=0sec 2024-11-11T13:37:19,271 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:19,271 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:19,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742322_1498 (size=12301) 2024-11-11T13:37:19,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/ed18a0debea04b20a29a8dc5e40a2f2a 2024-11-11T13:37:19,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332299278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/c9840ea42a80421ab4beff98e8a40d40 is 50, key is test_row_0/C:col10/1731332238297/Put/seqid=0 2024-11-11T13:37:19,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332299292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T13:37:19,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742324_1500 (size=12301) 2024-11-11T13:37:19,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/c9840ea42a80421ab4beff98e8a40d40 2024-11-11T13:37:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T13:37:19,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/86d0018787c6447381ed4c03244c929f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/86d0018787c6447381ed4c03244c929f 2024-11-11T13:37:19,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/86d0018787c6447381ed4c03244c929f, entries=200, sequenceid=364, filesize=14.4 K 2024-11-11T13:37:19,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/ed18a0debea04b20a29a8dc5e40a2f2a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/ed18a0debea04b20a29a8dc5e40a2f2a 2024-11-11T13:37:19,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/ed18a0debea04b20a29a8dc5e40a2f2a, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T13:37:19,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/c9840ea42a80421ab4beff98e8a40d40 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/c9840ea42a80421ab4beff98e8a40d40 2024-11-11T13:37:19,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/c9840ea42a80421ab4beff98e8a40d40, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T13:37:19,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ed96da846868b1163b5c13dff6f86179 in 221ms, sequenceid=364, compaction requested=false 2024-11-11T13:37:19,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:19,408 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:19,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T13:37:19,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/acc29fab29c84b1bba483f57b146c39b is 50, key is test_row_0/A:col10/1731332239241/Put/seqid=0 2024-11-11T13:37:19,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742325_1501 (size=12301) 2024-11-11T13:37:19,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/acc29fab29c84b1bba483f57b146c39b 2024-11-11T13:37:19,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a1062dd66e344c959064654b6ba3faf8 is 50, key is test_row_0/B:col10/1731332239241/Put/seqid=0 2024-11-11T13:37:19,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T13:37:19,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:19,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742326_1502 (size=12301) 2024-11-11T13:37:19,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332299512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332299514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332299520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332299521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33852 deadline: 1731332299569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,575 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in ed96da846868b1163b5c13dff6f86179 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,613 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/0285cc7eeb4f409586b924850abcbdf3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/0285cc7eeb4f409586b924850abcbdf3 2024-11-11T13:37:19,616 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into 0285cc7eeb4f409586b924850abcbdf3(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:19,616 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:19,616 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=13, startTime=1731332239116; duration=0sec 2024-11-11T13:37:19,617 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:19,617 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:19,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T13:37:19,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:19,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332299629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332299629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332299639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332299639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T13:37:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T13:37:19,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332299840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332299840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332299859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332299859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:19,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a1062dd66e344c959064654b6ba3faf8 2024-11-11T13:37:19,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/f31a9a4d5c7c412d8cd9f84ec8223afd is 50, key is test_row_0/C:col10/1731332239241/Put/seqid=0 2024-11-11T13:37:19,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742327_1503 (size=12301) 2024-11-11T13:37:19,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/f31a9a4d5c7c412d8cd9f84ec8223afd 2024-11-11T13:37:19,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/acc29fab29c84b1bba483f57b146c39b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/acc29fab29c84b1bba483f57b146c39b 2024-11-11T13:37:19,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/acc29fab29c84b1bba483f57b146c39b, entries=150, sequenceid=381, filesize=12.0 K 2024-11-11T13:37:19,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a1062dd66e344c959064654b6ba3faf8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a1062dd66e344c959064654b6ba3faf8 2024-11-11T13:37:19,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a1062dd66e344c959064654b6ba3faf8, entries=150, sequenceid=381, filesize=12.0 K 2024-11-11T13:37:19,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/f31a9a4d5c7c412d8cd9f84ec8223afd as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f31a9a4d5c7c412d8cd9f84ec8223afd 2024-11-11T13:37:19,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f31a9a4d5c7c412d8cd9f84ec8223afd, entries=150, sequenceid=381, filesize=12.0 K 2024-11-11T13:37:19,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ed96da846868b1163b5c13dff6f86179 in 510ms, sequenceid=381, compaction requested=true 2024-11-11T13:37:19,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:19,918 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:19,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:19,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:19,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:19,919 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:19,919 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:19,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:19,919 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:19,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:19,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:19,919 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,920 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92fc34e9d7714d5185de5e841e212bc1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/86d0018787c6447381ed4c03244c929f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/acc29fab29c84b1bba483f57b146c39b] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=39.2 K 2024-11-11T13:37:19,920 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:19,920 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:19,920 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,920 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/0285cc7eeb4f409586b924850abcbdf3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/ed18a0debea04b20a29a8dc5e40a2f2a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a1062dd66e344c959064654b6ba3faf8] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=36.8 K 2024-11-11T13:37:19,920 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 92fc34e9d7714d5185de5e841e212bc1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731332236982 2024-11-11T13:37:19,920 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0285cc7eeb4f409586b924850abcbdf3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731332236982 2024-11-11T13:37:19,920 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 86d0018787c6447381ed4c03244c929f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731332238236 2024-11-11T13:37:19,920 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed18a0debea04b20a29a8dc5e40a2f2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731332238294 2024-11-11T13:37:19,920 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting acc29fab29c84b1bba483f57b146c39b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731332239241 2024-11-11T13:37:19,920 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1062dd66e344c959064654b6ba3faf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731332239241 2024-11-11T13:37:19,929 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:19,930 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/66c9af7673bd424ba6cb0567b563f243 is 50, key is test_row_0/B:col10/1731332239241/Put/seqid=0 2024-11-11T13:37:19,932 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:19,933 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/4e3773c93520431c97938363c2e274ad is 50, key is test_row_0/A:col10/1731332239241/Put/seqid=0 2024-11-11T13:37:19,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742328_1504 (size=13221) 2024-11-11T13:37:19,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T13:37:19,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,942 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T13:37:19,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742329_1505 (size=13221) 2024-11-11T13:37:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/a466058f6e8f4b1880b78ef752c8fdd5 is 50, key is test_row_0/A:col10/1731332239505/Put/seqid=0 2024-11-11T13:37:19,956 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/4e3773c93520431c97938363c2e274ad as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/4e3773c93520431c97938363c2e274ad 2024-11-11T13:37:19,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742330_1506 (size=12301) 2024-11-11T13:37:19,964 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into 4e3773c93520431c97938363c2e274ad(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:19,964 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/a466058f6e8f4b1880b78ef752c8fdd5 2024-11-11T13:37:19,964 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:19,965 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=13, startTime=1731332239918; duration=0sec 2024-11-11T13:37:19,965 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:19,965 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:19,965 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:19,965 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:19,965 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:19,965 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:19,966 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f62d493aac044044aa22c898a1286662, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/c9840ea42a80421ab4beff98e8a40d40, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f31a9a4d5c7c412d8cd9f84ec8223afd] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=36.8 K 2024-11-11T13:37:19,966 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f62d493aac044044aa22c898a1286662, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731332236982 2024-11-11T13:37:19,967 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c9840ea42a80421ab4beff98e8a40d40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731332238294 2024-11-11T13:37:19,969 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f31a9a4d5c7c412d8cd9f84ec8223afd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731332239241 2024-11-11T13:37:19,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/83f57e34c2914ecfab16384f9e78b9c5 is 50, key is test_row_0/B:col10/1731332239505/Put/seqid=0 2024-11-11T13:37:19,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742331_1507 (size=12301) 2024-11-11T13:37:19,995 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/83f57e34c2914ecfab16384f9e78b9c5 2024-11-11T13:37:20,001 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:20,002 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/267736de5db94bb99f730afb41c881f3 is 50, key is test_row_0/C:col10/1731332239241/Put/seqid=0 2024-11-11T13:37:20,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742332_1508 (size=13221) 2024-11-11T13:37:20,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/dba12da380294ef4b1e4345679b620d0 is 50, key is test_row_0/C:col10/1731332239505/Put/seqid=0 2024-11-11T13:37:20,018 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/267736de5db94bb99f730afb41c881f3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/267736de5db94bb99f730afb41c881f3 2024-11-11T13:37:20,022 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into 267736de5db94bb99f730afb41c881f3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:20,023 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:20,023 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=13, startTime=1731332239919; duration=0sec 2024-11-11T13:37:20,023 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:20,023 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:20,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742333_1509 (size=12301) 2024-11-11T13:37:20,029 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/dba12da380294ef4b1e4345679b620d0 2024-11-11T13:37:20,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/a466058f6e8f4b1880b78ef752c8fdd5 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a466058f6e8f4b1880b78ef752c8fdd5 2024-11-11T13:37:20,039 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a466058f6e8f4b1880b78ef752c8fdd5, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T13:37:20,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/83f57e34c2914ecfab16384f9e78b9c5 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/83f57e34c2914ecfab16384f9e78b9c5 2024-11-11T13:37:20,042 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/83f57e34c2914ecfab16384f9e78b9c5, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T13:37:20,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/dba12da380294ef4b1e4345679b620d0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/dba12da380294ef4b1e4345679b620d0 2024-11-11T13:37:20,072 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/dba12da380294ef4b1e4345679b620d0, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T13:37:20,073 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for ed96da846868b1163b5c13dff6f86179 in 131ms, sequenceid=403, compaction requested=false 2024-11-11T13:37:20,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:20,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:20,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-11T13:37:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-11T13:37:20,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T13:37:20,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3730 sec 2024-11-11T13:37:20,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3780 sec 2024-11-11T13:37:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:20,170 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:20,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:37:20,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/5896cb69dd864b0bb6ae34b735942a71 is 50, key is test_row_0/A:col10/1731332240163/Put/seqid=0 2024-11-11T13:37:20,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742334_1510 (size=14741) 2024-11-11T13:37:20,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/5896cb69dd864b0bb6ae34b735942a71 2024-11-11T13:37:20,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/85474b8c799544a690344b7201b58a3b is 50, key is test_row_0/B:col10/1731332240163/Put/seqid=0 2024-11-11T13:37:20,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742335_1511 (size=12301) 2024-11-11T13:37:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332300221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332300222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332300225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332300225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332300329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332300339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332300339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332300342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,355 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/66c9af7673bd424ba6cb0567b563f243 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/66c9af7673bd424ba6cb0567b563f243 2024-11-11T13:37:20,358 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into 66c9af7673bd424ba6cb0567b563f243(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:20,358 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:20,358 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=13, startTime=1731332239919; duration=0sec 2024-11-11T13:37:20,358 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:20,358 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:20,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332300550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332300550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332300551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332300559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/85474b8c799544a690344b7201b58a3b 2024-11-11T13:37:20,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/b4ec693a35e3468281f42630e9212fec is 50, key is test_row_0/C:col10/1731332240163/Put/seqid=0 2024-11-11T13:37:20,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742336_1512 (size=12301) 2024-11-11T13:37:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T13:37:20,849 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:20,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-11T13:37:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T13:37:20,851 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:20,851 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:20,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:20,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332300859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332300859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332300869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332300879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T13:37:21,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T13:37:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:21,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:21,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:21,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/b4ec693a35e3468281f42630e9212fec 2024-11-11T13:37:21,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/5896cb69dd864b0bb6ae34b735942a71 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5896cb69dd864b0bb6ae34b735942a71 2024-11-11T13:37:21,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5896cb69dd864b0bb6ae34b735942a71, entries=200, sequenceid=416, filesize=14.4 K 2024-11-11T13:37:21,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/85474b8c799544a690344b7201b58a3b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/85474b8c799544a690344b7201b58a3b 2024-11-11T13:37:21,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/85474b8c799544a690344b7201b58a3b, entries=150, sequenceid=416, filesize=12.0 K 2024-11-11T13:37:21,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/b4ec693a35e3468281f42630e9212fec as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/b4ec693a35e3468281f42630e9212fec 2024-11-11T13:37:21,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/b4ec693a35e3468281f42630e9212fec, entries=150, sequenceid=416, filesize=12.0 K 2024-11-11T13:37:21,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ed96da846868b1163b5c13dff6f86179 in 897ms, sequenceid=416, compaction requested=true 2024-11-11T13:37:21,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:37:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T13:37:21,068 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:21,068 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:21,069 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:21,069 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:21,069 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:21,069 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/267736de5db94bb99f730afb41c881f3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/dba12da380294ef4b1e4345679b620d0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/b4ec693a35e3468281f42630e9212fec] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=36.9 K 2024-11-11T13:37:21,069 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:21,069 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:21,069 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:21,070 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/4e3773c93520431c97938363c2e274ad, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a466058f6e8f4b1880b78ef752c8fdd5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5896cb69dd864b0bb6ae34b735942a71] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=39.3 K 2024-11-11T13:37:21,070 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 267736de5db94bb99f730afb41c881f3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731332239241 2024-11-11T13:37:21,070 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e3773c93520431c97938363c2e274ad, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731332239241 2024-11-11T13:37:21,070 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a466058f6e8f4b1880b78ef752c8fdd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731332239505 2024-11-11T13:37:21,070 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting dba12da380294ef4b1e4345679b620d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731332239505 2024-11-11T13:37:21,071 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5896cb69dd864b0bb6ae34b735942a71, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731332240152 2024-11-11T13:37:21,071 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ec693a35e3468281f42630e9212fec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731332240152 2024-11-11T13:37:21,082 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:21,082 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/5a9ed34bd64b4c2f8d2031d3eca5a316 is 50, key is test_row_0/A:col10/1731332240163/Put/seqid=0 2024-11-11T13:37:21,083 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:21,084 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/5769ab21b45f434d995486ab53c44ab8 is 50, key is test_row_0/C:col10/1731332240163/Put/seqid=0 2024-11-11T13:37:21,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742337_1513 (size=13323) 2024-11-11T13:37:21,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742338_1514 (size=13323) 2024-11-11T13:37:21,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T13:37:21,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:21,160 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:37:21,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/182d8675f34f4301abb093de03efe116 is 50, key is test_row_0/A:col10/1731332240188/Put/seqid=0 2024-11-11T13:37:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T13:37:21,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742339_1515 (size=12301) 2024-11-11T13:37:21,189 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/182d8675f34f4301abb093de03efe116 2024-11-11T13:37:21,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/7342891b704448d4aa9fde2fc42f56e6 is 50, key is test_row_0/B:col10/1731332240188/Put/seqid=0 2024-11-11T13:37:21,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742340_1516 (size=12301) 2024-11-11T13:37:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:21,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:21,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332301390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332301396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332301400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332301400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T13:37:21,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332301509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332301520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332301520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332301520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,548 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/5769ab21b45f434d995486ab53c44ab8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/5769ab21b45f434d995486ab53c44ab8 2024-11-11T13:37:21,553 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/5a9ed34bd64b4c2f8d2031d3eca5a316 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5a9ed34bd64b4c2f8d2031d3eca5a316 2024-11-11T13:37:21,564 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into 5769ab21b45f434d995486ab53c44ab8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:21,564 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:21,564 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=13, startTime=1731332241068; duration=0sec 2024-11-11T13:37:21,564 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:21,564 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:21,564 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:21,564 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into 5a9ed34bd64b4c2f8d2031d3eca5a316(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:21,565 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:21,565 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=13, startTime=1731332241067; duration=0sec 2024-11-11T13:37:21,565 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:21,565 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:21,565 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:21,565 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:21,565 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:21,565 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/66c9af7673bd424ba6cb0567b563f243, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/83f57e34c2914ecfab16384f9e78b9c5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/85474b8c799544a690344b7201b58a3b] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=36.9 K 2024-11-11T13:37:21,566 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 66c9af7673bd424ba6cb0567b563f243, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731332239241 2024-11-11T13:37:21,566 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f57e34c2914ecfab16384f9e78b9c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731332239505 2024-11-11T13:37:21,566 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 85474b8c799544a690344b7201b58a3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731332240152 2024-11-11T13:37:21,572 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:21,573 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/38c432b1cd994aa4ba5faf87a60bf89c is 50, key is test_row_0/B:col10/1731332240163/Put/seqid=0 2024-11-11T13:37:21,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742341_1517 (size=13323) 2024-11-11T13:37:21,624 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/7342891b704448d4aa9fde2fc42f56e6 2024-11-11T13:37:21,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/f898484e189c43fab1c24d36e4f7b715 is 50, key is test_row_0/C:col10/1731332240188/Put/seqid=0 2024-11-11T13:37:21,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742342_1518 (size=12301) 2024-11-11T13:37:21,712 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/f898484e189c43fab1c24d36e4f7b715 2024-11-11T13:37:21,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/182d8675f34f4301abb093de03efe116 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/182d8675f34f4301abb093de03efe116 2024-11-11T13:37:21,725 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/182d8675f34f4301abb093de03efe116, entries=150, sequenceid=442, filesize=12.0 K 2024-11-11T13:37:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/7342891b704448d4aa9fde2fc42f56e6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/7342891b704448d4aa9fde2fc42f56e6 2024-11-11T13:37:21,728 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/7342891b704448d4aa9fde2fc42f56e6, entries=150, sequenceid=442, filesize=12.0 K 2024-11-11T13:37:21,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/f898484e189c43fab1c24d36e4f7b715 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f898484e189c43fab1c24d36e4f7b715 2024-11-11T13:37:21,732 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f898484e189c43fab1c24d36e4f7b715, entries=150, sequenceid=442, filesize=12.0 K 2024-11-11T13:37:21,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332301729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,733 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ed96da846868b1163b5c13dff6f86179 in 574ms, sequenceid=442, compaction requested=false 2024-11-11T13:37:21,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:21,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:21,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-11T13:37:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-11T13:37:21,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T13:37:21,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 882 msec 2024-11-11T13:37:21,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 887 msec 2024-11-11T13:37:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:21,743 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:21,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T13:37:21,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/12ec9829f0244c9486bf9ca03b80411a is 50, key is test_row_0/A:col10/1731332241743/Put/seqid=0 2024-11-11T13:37:21,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742343_1519 (size=17181) 2024-11-11T13:37:21,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/12ec9829f0244c9486bf9ca03b80411a 2024-11-11T13:37:21,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/4f752322d18f4b1988a0aa8df28c18c6 is 50, key is test_row_0/B:col10/1731332241743/Put/seqid=0 2024-11-11T13:37:21,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742344_1520 (size=12301) 2024-11-11T13:37:21,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332301785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332301785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332301788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332301899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332301899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332301899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T13:37:21,989 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:21,990 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-11T13:37:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T13:37:21,991 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:21,992 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:21,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:22,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332302039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,048 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/38c432b1cd994aa4ba5faf87a60bf89c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/38c432b1cd994aa4ba5faf87a60bf89c 2024-11-11T13:37:22,053 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into 38c432b1cd994aa4ba5faf87a60bf89c(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:22,053 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:22,053 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=13, startTime=1731332241068; duration=0sec 2024-11-11T13:37:22,053 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:22,053 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T13:37:22,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332302109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332302109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332302109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T13:37:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:22,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:22,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/4f752322d18f4b1988a0aa8df28c18c6 2024-11-11T13:37:22,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/5eefd1e0681e484b801647342795e351 is 50, key is test_row_0/C:col10/1731332241743/Put/seqid=0 2024-11-11T13:37:22,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742345_1521 (size=12301) 2024-11-11T13:37:22,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/5eefd1e0681e484b801647342795e351 2024-11-11T13:37:22,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/12ec9829f0244c9486bf9ca03b80411a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/12ec9829f0244c9486bf9ca03b80411a 2024-11-11T13:37:22,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/12ec9829f0244c9486bf9ca03b80411a, entries=250, sequenceid=457, filesize=16.8 K 2024-11-11T13:37:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/4f752322d18f4b1988a0aa8df28c18c6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/4f752322d18f4b1988a0aa8df28c18c6 2024-11-11T13:37:22,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/4f752322d18f4b1988a0aa8df28c18c6, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T13:37:22,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/5eefd1e0681e484b801647342795e351 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/5eefd1e0681e484b801647342795e351 2024-11-11T13:37:22,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/5eefd1e0681e484b801647342795e351, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T13:37:22,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ed96da846868b1163b5c13dff6f86179 in 490ms, sequenceid=457, compaction requested=true 2024-11-11T13:37:22,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:22,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:22,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:22,233 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:22,233 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:22,235 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:22,235 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:22,235 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:22,235 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/38c432b1cd994aa4ba5faf87a60bf89c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/7342891b704448d4aa9fde2fc42f56e6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/4f752322d18f4b1988a0aa8df28c18c6] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=37.0 K 2024-11-11T13:37:22,238 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:22,238 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:22,238 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:22,239 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5a9ed34bd64b4c2f8d2031d3eca5a316, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/182d8675f34f4301abb093de03efe116, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/12ec9829f0244c9486bf9ca03b80411a] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=41.8 K 2024-11-11T13:37:22,239 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c432b1cd994aa4ba5faf87a60bf89c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731332240152 2024-11-11T13:37:22,239 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a9ed34bd64b4c2f8d2031d3eca5a316, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731332240152 2024-11-11T13:37:22,239 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 7342891b704448d4aa9fde2fc42f56e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731332240188 2024-11-11T13:37:22,239 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 182d8675f34f4301abb093de03efe116, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731332240188 2024-11-11T13:37:22,240 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f752322d18f4b1988a0aa8df28c18c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731332241398 2024-11-11T13:37:22,240 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12ec9829f0244c9486bf9ca03b80411a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731332241388 2024-11-11T13:37:22,253 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:22,254 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a4ed7a064799410aaee72c5e8e762936 is 50, key is test_row_0/B:col10/1731332241743/Put/seqid=0 2024-11-11T13:37:22,256 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:22,256 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/b31f713df8564854a0636a447b132c95 is 50, key is test_row_0/A:col10/1731332241743/Put/seqid=0 2024-11-11T13:37:22,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742346_1522 (size=13425) 2024-11-11T13:37:22,285 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a4ed7a064799410aaee72c5e8e762936 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a4ed7a064799410aaee72c5e8e762936 2024-11-11T13:37:22,295 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into a4ed7a064799410aaee72c5e8e762936(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:22,295 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:22,295 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=13, startTime=1731332242233; duration=0sec 2024-11-11T13:37:22,295 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:22,295 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:22,296 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:22,297 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:22,297 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:22,297 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:22,297 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/5769ab21b45f434d995486ab53c44ab8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f898484e189c43fab1c24d36e4f7b715, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/5eefd1e0681e484b801647342795e351] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=37.0 K 2024-11-11T13:37:22,298 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 5769ab21b45f434d995486ab53c44ab8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731332240152 2024-11-11T13:37:22,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T13:37:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:22,300 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T13:37:22,301 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f898484e189c43fab1c24d36e4f7b715, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731332240188 2024-11-11T13:37:22,301 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eefd1e0681e484b801647342795e351, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731332241398 2024-11-11T13:37:22,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742347_1523 (size=13425) 2024-11-11T13:37:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T13:37:22,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/e5c6c7b7be464928b8e070f4f55db916 is 50, key is test_row_0/A:col10/1731332241766/Put/seqid=0 2024-11-11T13:37:22,333 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:22,333 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/1b9ed88af06e421dabb1de49f94e702f is 50, key is test_row_0/C:col10/1731332241743/Put/seqid=0 2024-11-11T13:37:22,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742348_1524 (size=12301) 2024-11-11T13:37:22,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742349_1525 (size=13425) 2024-11-11T13:37:22,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:22,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332302453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332302459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332302460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332302551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332302569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332302579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332302579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T13:37:22,742 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/b31f713df8564854a0636a447b132c95 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b31f713df8564854a0636a447b132c95 2024-11-11T13:37:22,743 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/e5c6c7b7be464928b8e070f4f55db916 2024-11-11T13:37:22,765 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/1b9ed88af06e421dabb1de49f94e702f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/1b9ed88af06e421dabb1de49f94e702f 2024-11-11T13:37:22,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/b02f1d421607486bad1921e254ee1abe is 50, key is test_row_0/B:col10/1731332241766/Put/seqid=0 2024-11-11T13:37:22,777 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into b31f713df8564854a0636a447b132c95(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:22,777 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:22,777 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=13, startTime=1731332242233; duration=0sec 2024-11-11T13:37:22,777 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:22,777 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:22,777 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into 1b9ed88af06e421dabb1de49f94e702f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:22,777 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:22,777 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=13, startTime=1731332242234; duration=0sec 2024-11-11T13:37:22,777 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:22,777 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:22,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742350_1526 (size=12301) 2024-11-11T13:37:22,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332302780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332302789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:22,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332302799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:22,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332303099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332303099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332303109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T13:37:23,181 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/b02f1d421607486bad1921e254ee1abe 2024-11-11T13:37:23,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/28381b1ec9464fb4a6be18c5e407dcab is 50, key is test_row_0/C:col10/1731332241766/Put/seqid=0 2024-11-11T13:37:23,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742351_1527 (size=12301) 2024-11-11T13:37:23,203 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/28381b1ec9464fb4a6be18c5e407dcab 2024-11-11T13:37:23,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/e5c6c7b7be464928b8e070f4f55db916 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/e5c6c7b7be464928b8e070f4f55db916 2024-11-11T13:37:23,214 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/e5c6c7b7be464928b8e070f4f55db916, entries=150, sequenceid=482, filesize=12.0 K 2024-11-11T13:37:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/b02f1d421607486bad1921e254ee1abe as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b02f1d421607486bad1921e254ee1abe 2024-11-11T13:37:23,220 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b02f1d421607486bad1921e254ee1abe, entries=150, sequenceid=482, filesize=12.0 K 2024-11-11T13:37:23,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/28381b1ec9464fb4a6be18c5e407dcab as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/28381b1ec9464fb4a6be18c5e407dcab 2024-11-11T13:37:23,236 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/28381b1ec9464fb4a6be18c5e407dcab, entries=150, sequenceid=482, filesize=12.0 K 2024-11-11T13:37:23,237 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ed96da846868b1163b5c13dff6f86179 in 937ms, sequenceid=482, compaction requested=false 2024-11-11T13:37:23,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:23,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:23,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T13:37:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-11T13:37:23,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T13:37:23,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2470 sec 2024-11-11T13:37:23,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.2530 sec 2024-11-11T13:37:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:23,572 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:23,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T13:37:23,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/b6297b3a7f0b4ce8ae3e1d5d99e36c54 is 50, key is test_row_0/A:col10/1731332242445/Put/seqid=0 2024-11-11T13:37:23,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742352_1528 (size=14741) 2024-11-11T13:37:23,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/b6297b3a7f0b4ce8ae3e1d5d99e36c54 2024-11-11T13:37:23,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a3acdb1e91554090aee5426e510f2d52 is 50, key is test_row_0/B:col10/1731332242445/Put/seqid=0 2024-11-11T13:37:23,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742353_1529 (size=12301) 2024-11-11T13:37:23,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a3acdb1e91554090aee5426e510f2d52 2024-11-11T13:37:23,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/76e5c861f4b340b79ae37429731a9587 is 50, key is test_row_0/C:col10/1731332242445/Put/seqid=0 2024-11-11T13:37:23,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742354_1530 (size=12301) 2024-11-11T13:37:23,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/76e5c861f4b340b79ae37429731a9587 2024-11-11T13:37:23,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/b6297b3a7f0b4ce8ae3e1d5d99e36c54 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b6297b3a7f0b4ce8ae3e1d5d99e36c54 2024-11-11T13:37:23,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332303649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b6297b3a7f0b4ce8ae3e1d5d99e36c54, entries=200, sequenceid=498, filesize=14.4 K 2024-11-11T13:37:23,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a3acdb1e91554090aee5426e510f2d52 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a3acdb1e91554090aee5426e510f2d52 2024-11-11T13:37:23,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332303655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332303657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a3acdb1e91554090aee5426e510f2d52, entries=150, sequenceid=498, filesize=12.0 K 2024-11-11T13:37:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332303659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/76e5c861f4b340b79ae37429731a9587 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/76e5c861f4b340b79ae37429731a9587 2024-11-11T13:37:23,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/76e5c861f4b340b79ae37429731a9587, entries=150, sequenceid=498, filesize=12.0 K 2024-11-11T13:37:23,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ed96da846868b1163b5c13dff6f86179 in 102ms, sequenceid=498, compaction requested=true 2024-11-11T13:37:23,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:23,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:23,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:23,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:23,675 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:23,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:23,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:23,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:37:23,675 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:23,676 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:23,676 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:23,676 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:23,676 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:23,676 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:23,676 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b31f713df8564854a0636a447b132c95, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/e5c6c7b7be464928b8e070f4f55db916, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b6297b3a7f0b4ce8ae3e1d5d99e36c54] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=39.5 K 2024-11-11T13:37:23,676 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:23,676 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a4ed7a064799410aaee72c5e8e762936, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b02f1d421607486bad1921e254ee1abe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a3acdb1e91554090aee5426e510f2d52] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=37.1 K 2024-11-11T13:37:23,677 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting b31f713df8564854a0636a447b132c95, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731332241398 2024-11-11T13:37:23,677 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4ed7a064799410aaee72c5e8e762936, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731332241398 2024-11-11T13:37:23,677 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e5c6c7b7be464928b8e070f4f55db916, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1731332241766 2024-11-11T13:37:23,677 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting b02f1d421607486bad1921e254ee1abe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1731332241766 2024-11-11T13:37:23,677 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting b6297b3a7f0b4ce8ae3e1d5d99e36c54, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731332242445 2024-11-11T13:37:23,677 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3acdb1e91554090aee5426e510f2d52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731332242445 2024-11-11T13:37:23,709 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:23,710 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/2fb9468b16c74876b0d16462cacdfbb7 is 50, key is test_row_0/B:col10/1731332242445/Put/seqid=0 2024-11-11T13:37:23,729 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:23,730 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/46140763acdc436893769947a0d6b664 is 50, key is test_row_0/A:col10/1731332242445/Put/seqid=0 2024-11-11T13:37:23,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742355_1531 (size=13527) 2024-11-11T13:37:23,749 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/2fb9468b16c74876b0d16462cacdfbb7 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/2fb9468b16c74876b0d16462cacdfbb7 2024-11-11T13:37:23,754 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into 2fb9468b16c74876b0d16462cacdfbb7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:23,754 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:23,754 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=13, startTime=1731332243675; duration=0sec 2024-11-11T13:37:23,754 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:23,754 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:23,754 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:23,759 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:23,759 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:23,759 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:23,759 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/1b9ed88af06e421dabb1de49f94e702f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/28381b1ec9464fb4a6be18c5e407dcab, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/76e5c861f4b340b79ae37429731a9587] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=37.1 K 2024-11-11T13:37:23,759 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b9ed88af06e421dabb1de49f94e702f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731332241398 2024-11-11T13:37:23,760 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28381b1ec9464fb4a6be18c5e407dcab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=482, earliestPutTs=1731332241766 2024-11-11T13:37:23,760 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76e5c861f4b340b79ae37429731a9587, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731332242445 2024-11-11T13:37:23,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742356_1532 (size=13527) 2024-11-11T13:37:23,772 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/46140763acdc436893769947a0d6b664 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/46140763acdc436893769947a0d6b664 2024-11-11T13:37:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:23,779 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:23,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T13:37:23,781 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:23,782 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/6a3d8f955ba44a9fb4b5cf7b5b6d9592 is 50, key is test_row_0/C:col10/1731332242445/Put/seqid=0 2024-11-11T13:37:23,782 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into 46140763acdc436893769947a0d6b664(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:23,782 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:23,782 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=13, startTime=1731332243674; duration=0sec 2024-11-11T13:37:23,782 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:23,782 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:23,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742357_1533 (size=13527) 2024-11-11T13:37:23,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/74e7325e67134cbda8679b0b6031eddc is 50, key is test_row_0/A:col10/1731332243774/Put/seqid=0 2024-11-11T13:37:23,820 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/6a3d8f955ba44a9fb4b5cf7b5b6d9592 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/6a3d8f955ba44a9fb4b5cf7b5b6d9592 2024-11-11T13:37:23,824 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into 6a3d8f955ba44a9fb4b5cf7b5b6d9592(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:23,824 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:23,824 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=13, startTime=1731332243675; duration=0sec 2024-11-11T13:37:23,824 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:23,824 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:23,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332303810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332303816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332303817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332303817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742358_1534 (size=14741) 2024-11-11T13:37:23,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332303932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332303935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332303939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:23,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332303941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:23,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T13:37:24,142 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:24,142 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:24,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332304141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-11T13:37:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T13:37:24,156 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:24,157 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:24,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:24,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332304160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332304160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332304163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/74e7325e67134cbda8679b0b6031eddc 2024-11-11T13:37:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T13:37:24,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a4a0222bc4a148368ceddbd3d8b753ae is 50, key is test_row_0/B:col10/1731332243774/Put/seqid=0 2024-11-11T13:37:24,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742359_1535 (size=12301) 2024-11-11T13:37:24,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a4a0222bc4a148368ceddbd3d8b753ae 2024-11-11T13:37:24,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/11e7a409e2cf4da6802e0aa5a3a5f0d1 is 50, key is test_row_0/C:col10/1731332243774/Put/seqid=0 2024-11-11T13:37:24,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T13:37:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:24,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742360_1536 (size=12301) 2024-11-11T13:37:24,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/11e7a409e2cf4da6802e0aa5a3a5f0d1 2024-11-11T13:37:24,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/74e7325e67134cbda8679b0b6031eddc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/74e7325e67134cbda8679b0b6031eddc 2024-11-11T13:37:24,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/74e7325e67134cbda8679b0b6031eddc, entries=200, sequenceid=522, filesize=14.4 K 2024-11-11T13:37:24,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/a4a0222bc4a148368ceddbd3d8b753ae as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a4a0222bc4a148368ceddbd3d8b753ae 2024-11-11T13:37:24,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a4a0222bc4a148368ceddbd3d8b753ae, entries=150, sequenceid=522, filesize=12.0 K 2024-11-11T13:37:24,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/11e7a409e2cf4da6802e0aa5a3a5f0d1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/11e7a409e2cf4da6802e0aa5a3a5f0d1 2024-11-11T13:37:24,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/11e7a409e2cf4da6802e0aa5a3a5f0d1, entries=150, sequenceid=522, filesize=12.0 K 2024-11-11T13:37:24,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ed96da846868b1163b5c13dff6f86179 in 617ms, sequenceid=522, compaction requested=false 2024-11-11T13:37:24,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T13:37:24,474 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:24,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T13:37:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:24,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T13:37:24,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:24,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:24,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:24,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/f2dca79f6739486a83c541a4897ef369 is 50, key is test_row_0/A:col10/1731332243814/Put/seqid=0 2024-11-11T13:37:24,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742361_1537 (size=14741) 2024-11-11T13:37:24,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332304538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332304538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332304539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332304539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T13:37:24,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:24,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:24,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:24,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,665 INFO [Thread-1927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1927. 2024-11-11T13:37:24,665 DEBUG [Thread-1927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:24,665 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:24,665 INFO [Thread-1919 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1919. 2024-11-11T13:37:24,665 DEBUG [Thread-1919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:24,665 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:24,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,665 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332304660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332304660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332304662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332304662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,668 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:24,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:24,669 INFO [Thread-1923 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1923. 2024-11-11T13:37:24,669 DEBUG [Thread-1923 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:24,669 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:24,669 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:24,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:24,675 INFO [Thread-1921 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1921. 2024-11-11T13:37:24,675 DEBUG [Thread-1921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:24,675 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:24,675 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:24,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:24,677 INFO [Thread-1925 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1925. 2024-11-11T13:37:24,677 DEBUG [Thread-1925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:24,677 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:24,677 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:24,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T13:37:24,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T13:37:24,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:24,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:24,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:24,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332304869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332304869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332304869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332304880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:24,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:24,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/f2dca79f6739486a83c541a4897ef369 2024-11-11T13:37:24,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/b24ebc170c3e42ff94d2aa5000eb588e is 50, key is test_row_0/B:col10/1731332243814/Put/seqid=0 2024-11-11T13:37:24,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742362_1538 (size=12301) 2024-11-11T13:37:24,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T13:37:24,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:24,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:24,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:24,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T13:37:25,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:25,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:25,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332305180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332305181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:25,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:25,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332305181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:25,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332305189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:25,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T13:37:25,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:25,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T13:37:25,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/b24ebc170c3e42ff94d2aa5000eb588e 2024-11-11T13:37:25,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/f8830314989a44798e4caa51eeabea31 is 50, key is test_row_0/C:col10/1731332243814/Put/seqid=0 2024-11-11T13:37:25,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742363_1539 (size=12301) 2024-11-11T13:37:25,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T13:37:25,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:25,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T13:37:25,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:25,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:25,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:25,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33860 deadline: 1731332305689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33838 deadline: 1731332305689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33840 deadline: 1731332305689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:25,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33870 deadline: 1731332305699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 , the old value is region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., hostname=37d26ecf6460,36877,1731332101576, seqNum=2 because the exception is null or not the one we care about 2024-11-11T13:37:25,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T13:37:25,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:25,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:25,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/f8830314989a44798e4caa51eeabea31 2024-11-11T13:37:25,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/f2dca79f6739486a83c541a4897ef369 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/f2dca79f6739486a83c541a4897ef369 2024-11-11T13:37:25,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/f2dca79f6739486a83c541a4897ef369, entries=200, sequenceid=540, filesize=14.4 K 2024-11-11T13:37:25,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/b24ebc170c3e42ff94d2aa5000eb588e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b24ebc170c3e42ff94d2aa5000eb588e 2024-11-11T13:37:25,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b24ebc170c3e42ff94d2aa5000eb588e, entries=150, sequenceid=540, filesize=12.0 K 2024-11-11T13:37:25,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/f8830314989a44798e4caa51eeabea31 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f8830314989a44798e4caa51eeabea31 2024-11-11T13:37:25,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f8830314989a44798e4caa51eeabea31, entries=150, sequenceid=540, filesize=12.0 K 2024-11-11T13:37:25,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for ed96da846868b1163b5c13dff6f86179 in 1277ms, sequenceid=540, compaction requested=true 2024-11-11T13:37:25,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:25,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:25,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:25,751 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:25,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:25,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:25,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed96da846868b1163b5c13dff6f86179:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:25,751 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:25,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:25,752 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:25,752 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:25,752 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/A is initiating minor compaction (all files) 2024-11-11T13:37:25,752 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/B is initiating minor compaction (all files) 2024-11-11T13:37:25,752 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/A in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,752 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/B in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,752 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/2fb9468b16c74876b0d16462cacdfbb7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a4a0222bc4a148368ceddbd3d8b753ae, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b24ebc170c3e42ff94d2aa5000eb588e] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=37.2 K 2024-11-11T13:37:25,752 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/46140763acdc436893769947a0d6b664, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/74e7325e67134cbda8679b0b6031eddc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/f2dca79f6739486a83c541a4897ef369] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=42.0 K 2024-11-11T13:37:25,752 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fb9468b16c74876b0d16462cacdfbb7, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731332242445 2024-11-11T13:37:25,752 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46140763acdc436893769947a0d6b664, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731332242445 2024-11-11T13:37:25,753 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74e7325e67134cbda8679b0b6031eddc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731332243622 2024-11-11T13:37:25,753 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a0222bc4a148368ceddbd3d8b753ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731332243622 2024-11-11T13:37:25,753 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2dca79f6739486a83c541a4897ef369, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731332243814 2024-11-11T13:37:25,753 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting b24ebc170c3e42ff94d2aa5000eb588e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731332243814 2024-11-11T13:37:25,758 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:25,758 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#A#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:25,758 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/b2ec13172b134dc5aee2e36cec89f49f is 50, key is test_row_0/B:col10/1731332243814/Put/seqid=0 2024-11-11T13:37:25,758 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/b72ea696509d4347ab4ce26e62d53a15 is 50, key is test_row_0/A:col10/1731332243814/Put/seqid=0 2024-11-11T13:37:25,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742364_1540 (size=13629) 2024-11-11T13:37:25,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742365_1541 (size=13629) 2024-11-11T13:37:25,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T13:37:25,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:25,860 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T13:37:25,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/92228bfcca5d4663857241353325a784 is 50, key is test_row_0/A:col10/1731332244537/Put/seqid=0 2024-11-11T13:37:25,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742366_1542 (size=12301) 2024-11-11T13:37:26,172 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/b2ec13172b134dc5aee2e36cec89f49f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b2ec13172b134dc5aee2e36cec89f49f 2024-11-11T13:37:26,172 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/b72ea696509d4347ab4ce26e62d53a15 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b72ea696509d4347ab4ce26e62d53a15 2024-11-11T13:37:26,177 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/B of ed96da846868b1163b5c13dff6f86179 into b2ec13172b134dc5aee2e36cec89f49f(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:26,177 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/A of ed96da846868b1163b5c13dff6f86179 into b72ea696509d4347ab4ce26e62d53a15(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:26,177 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:26,177 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:26,177 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/A, priority=13, startTime=1731332245751; duration=0sec 2024-11-11T13:37:26,177 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/B, priority=13, startTime=1731332245751; duration=0sec 2024-11-11T13:37:26,177 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:26,177 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:B 2024-11-11T13:37:26,177 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:26,177 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:A 2024-11-11T13:37:26,177 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:26,178 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:26,178 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): ed96da846868b1163b5c13dff6f86179/C is initiating minor compaction (all files) 2024-11-11T13:37:26,178 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed96da846868b1163b5c13dff6f86179/C in TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:26,178 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/6a3d8f955ba44a9fb4b5cf7b5b6d9592, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/11e7a409e2cf4da6802e0aa5a3a5f0d1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f8830314989a44798e4caa51eeabea31] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp, totalSize=37.2 K 2024-11-11T13:37:26,178 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a3d8f955ba44a9fb4b5cf7b5b6d9592, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731332242445 2024-11-11T13:37:26,179 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e7a409e2cf4da6802e0aa5a3a5f0d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731332243622 2024-11-11T13:37:26,179 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f8830314989a44798e4caa51eeabea31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731332243814 2024-11-11T13:37:26,186 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed96da846868b1163b5c13dff6f86179#C#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:26,186 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/1a780cf804b74064a25dfa74a8c74ff2 is 50, key is test_row_0/C:col10/1731332243814/Put/seqid=0 2024-11-11T13:37:26,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742367_1543 (size=13629) 2024-11-11T13:37:26,272 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/92228bfcca5d4663857241353325a784 2024-11-11T13:37:26,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/1c1b751bedec487a8159d47ba4942760 is 50, key is test_row_0/B:col10/1731332244537/Put/seqid=0 2024-11-11T13:37:26,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742368_1544 (size=12301) 2024-11-11T13:37:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T13:37:26,602 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/1a780cf804b74064a25dfa74a8c74ff2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/1a780cf804b74064a25dfa74a8c74ff2 2024-11-11T13:37:26,606 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed96da846868b1163b5c13dff6f86179/C of ed96da846868b1163b5c13dff6f86179 into 1a780cf804b74064a25dfa74a8c74ff2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:26,606 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:26,606 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., storeName=ed96da846868b1163b5c13dff6f86179/C, priority=13, startTime=1731332245751; duration=0sec 2024-11-11T13:37:26,606 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:26,607 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed96da846868b1163b5c13dff6f86179:C 2024-11-11T13:37:26,690 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/1c1b751bedec487a8159d47ba4942760 2024-11-11T13:37:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:26,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. as already flushing 2024-11-11T13:37:26,701 INFO [Thread-1912 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1912. 2024-11-11T13:37:26,701 INFO [Thread-1908 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1908. 2024-11-11T13:37:26,701 DEBUG [Thread-1912 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:26,701 DEBUG [Thread-1908 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:26,701 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:26,701 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:26,701 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:26,701 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:26,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:26,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:26,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/3489aa78f4ef4cb99766a052b4c8b34d is 50, key is test_row_0/C:col10/1731332244537/Put/seqid=0 2024-11-11T13:37:26,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742369_1545 (size=12301) 2024-11-11T13:37:26,710 INFO [Thread-1916 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1916. 2024-11-11T13:37:26,710 INFO [Thread-1910 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1910. 2024-11-11T13:37:26,710 DEBUG [Thread-1910 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:26,710 DEBUG [Thread-1916 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:26,710 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:26,710 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:26,711 DEBUG [Thread-1910 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:26,711 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:26,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:26,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:27,106 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/3489aa78f4ef4cb99766a052b4c8b34d 2024-11-11T13:37:27,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/92228bfcca5d4663857241353325a784 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92228bfcca5d4663857241353325a784 2024-11-11T13:37:27,113 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92228bfcca5d4663857241353325a784, entries=150, sequenceid=560, filesize=12.0 K 2024-11-11T13:37:27,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/1c1b751bedec487a8159d47ba4942760 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1c1b751bedec487a8159d47ba4942760 2024-11-11T13:37:27,117 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1c1b751bedec487a8159d47ba4942760, entries=150, sequenceid=560, filesize=12.0 K 2024-11-11T13:37:27,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/3489aa78f4ef4cb99766a052b4c8b34d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/3489aa78f4ef4cb99766a052b4c8b34d 2024-11-11T13:37:27,120 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/3489aa78f4ef4cb99766a052b4c8b34d, entries=150, sequenceid=560, filesize=12.0 K 2024-11-11T13:37:27,121 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for ed96da846868b1163b5c13dff6f86179 in 1261ms, sequenceid=560, compaction requested=false 2024-11-11T13:37:27,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for ed96da846868b1163b5c13dff6f86179: 2024-11-11T13:37:27,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:27,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-11T13:37:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-11T13:37:27,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T13:37:27,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9660 sec 2024-11-11T13:37:27,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.9830 sec 2024-11-11T13:37:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T13:37:28,309 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:29,601 INFO [Thread-1914 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1914. 2024-11-11T13:37:29,601 DEBUG [Thread-1914 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:29,601 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:29,601 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:29,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:29,601 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T13:37:29,601 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-11T13:37:29,601 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-11T13:37:29,601 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-11T13:37:29,601 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-11T13:37:29,601 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-11T13:37:29,601 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T13:37:29,601 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T13:37:29,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1290 2024-11-11T13:37:29,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3870 rows 2024-11-11T13:37:29,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1296 2024-11-11T13:37:29,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3888 rows 2024-11-11T13:37:29,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1300 2024-11-11T13:37:29,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3900 rows 2024-11-11T13:37:29,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1296 2024-11-11T13:37:29,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3888 rows 2024-11-11T13:37:29,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1293 2024-11-11T13:37:29,602 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3879 rows 2024-11-11T13:37:29,602 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:37:29,602 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:37:29,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:29,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:29,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:29,603 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T13:37:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T13:37:29,605 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332249605"}]},"ts":"1731332249605"} 2024-11-11T13:37:29,606 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T13:37:29,606 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T13:37:29,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T13:37:29,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed96da846868b1163b5c13dff6f86179, UNASSIGN}] 2024-11-11T13:37:29,609 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed96da846868b1163b5c13dff6f86179, UNASSIGN 2024-11-11T13:37:29,609 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=ed96da846868b1163b5c13dff6f86179, regionState=CLOSING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:29,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed96da846868b1163b5c13dff6f86179, UNASSIGN because future has completed 2024-11-11T13:37:29,610 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T13:37:29,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:37:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T13:37:29,757 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T13:37:29,762 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:29,763 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T13:37:29,763 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing ed96da846868b1163b5c13dff6f86179, disabling compactions & flushes 2024-11-11T13:37:29,763 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:29,763 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:29,763 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. after waiting 0 ms 2024-11-11T13:37:29,763 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:29,763 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing ed96da846868b1163b5c13dff6f86179 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-11T13:37:29,767 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/bceb09b359b34559a834fc1b967bd145 is 50, key is test_row_0/A:col10/1731332249599/Put/seqid=0 2024-11-11T13:37:29,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742370_1546 (size=12301) 2024-11-11T13:37:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T13:37:30,177 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/bceb09b359b34559a834fc1b967bd145 2024-11-11T13:37:30,182 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/8819df35cbf14ecabdc4f3f2eb45f497 is 50, key is test_row_0/B:col10/1731332249599/Put/seqid=0 2024-11-11T13:37:30,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742371_1547 (size=12301) 2024-11-11T13:37:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T13:37:30,586 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/8819df35cbf14ecabdc4f3f2eb45f497 2024-11-11T13:37:30,592 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/06700996dada44ffbcec03c0306101f3 is 50, key is test_row_0/C:col10/1731332249599/Put/seqid=0 2024-11-11T13:37:30,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742372_1548 (size=12301) 2024-11-11T13:37:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T13:37:30,997 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/06700996dada44ffbcec03c0306101f3 2024-11-11T13:37:31,004 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/A/bceb09b359b34559a834fc1b967bd145 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/bceb09b359b34559a834fc1b967bd145 2024-11-11T13:37:31,009 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/bceb09b359b34559a834fc1b967bd145, entries=150, sequenceid=571, filesize=12.0 K 2024-11-11T13:37:31,010 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/B/8819df35cbf14ecabdc4f3f2eb45f497 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8819df35cbf14ecabdc4f3f2eb45f497 2024-11-11T13:37:31,013 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8819df35cbf14ecabdc4f3f2eb45f497, entries=150, sequenceid=571, filesize=12.0 K 2024-11-11T13:37:31,014 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/.tmp/C/06700996dada44ffbcec03c0306101f3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/06700996dada44ffbcec03c0306101f3 2024-11-11T13:37:31,017 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/06700996dada44ffbcec03c0306101f3, entries=150, sequenceid=571, filesize=12.0 K 2024-11-11T13:37:31,018 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ed96da846868b1163b5c13dff6f86179 in 1254ms, sequenceid=571, compaction requested=true 2024-11-11T13:37:31,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/9f39ff289f9e42319bc844b91b4160f1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/22b6d17050b94e1c9a4d585ef7fdfc72, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1d1fe98a2fb24a0c8110052206b4b855, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/43e1c22bc78d467ca0ab090520cf97bc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2d6b55b8c5e84d2ab51957095a96c5c4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2b8f8d39b24642f9856668676a884d41, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5f8dbf2de9a34ef691a5cecfd6bce783, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a54a22cf799142d9a9b962a6f6434361, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/00f050ae46124af5a17bab7ee5fb93e4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/33a54e7034294175909cabe02a59043d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1ce568dc4ac741e1b49f33481cc69232, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/156c5483c2a4420d911157895a89b129, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/ea9f7baa592f455781351c0c733e10ca, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/02bff4b5564f467c83db7762bd22515f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5bb24b160a064955a8b616a38edf3a1a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b75bfa54e40d4ab09d3d8d6aece0d04e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/d7d060c50dbf49de938a593b4233830d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/304536c32cf948aaaa314058d7290bfd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/4747057b0a79423bbddf568fea03e93b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92ce4cfe9bd44093be71e2aa5b57fa86, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/69b7b4433ff04a94b90713d5f17b27c9, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/0ef00d8928cc45928f36cdb24635391f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/136e528a903d471c9bc681ec692a791f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92cb1f1828444a708a924a07ae8498bd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92fc34e9d7714d5185de5e841e212bc1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/86d0018787c6447381ed4c03244c929f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/4e3773c93520431c97938363c2e274ad, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/acc29fab29c84b1bba483f57b146c39b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a466058f6e8f4b1880b78ef752c8fdd5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5896cb69dd864b0bb6ae34b735942a71, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5a9ed34bd64b4c2f8d2031d3eca5a316, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/182d8675f34f4301abb093de03efe116, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/12ec9829f0244c9486bf9ca03b80411a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b31f713df8564854a0636a447b132c95, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/e5c6c7b7be464928b8e070f4f55db916, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b6297b3a7f0b4ce8ae3e1d5d99e36c54, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/46140763acdc436893769947a0d6b664, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/74e7325e67134cbda8679b0b6031eddc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/f2dca79f6739486a83c541a4897ef369] to archive 2024-11-11T13:37:31,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:37:31,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/9f39ff289f9e42319bc844b91b4160f1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/9f39ff289f9e42319bc844b91b4160f1 2024-11-11T13:37:31,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/22b6d17050b94e1c9a4d585ef7fdfc72 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/22b6d17050b94e1c9a4d585ef7fdfc72 2024-11-11T13:37:31,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1d1fe98a2fb24a0c8110052206b4b855 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1d1fe98a2fb24a0c8110052206b4b855 2024-11-11T13:37:31,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/43e1c22bc78d467ca0ab090520cf97bc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/43e1c22bc78d467ca0ab090520cf97bc 2024-11-11T13:37:31,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2d6b55b8c5e84d2ab51957095a96c5c4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2d6b55b8c5e84d2ab51957095a96c5c4 2024-11-11T13:37:31,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2b8f8d39b24642f9856668676a884d41 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/2b8f8d39b24642f9856668676a884d41 2024-11-11T13:37:31,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5f8dbf2de9a34ef691a5cecfd6bce783 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5f8dbf2de9a34ef691a5cecfd6bce783 2024-11-11T13:37:31,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a54a22cf799142d9a9b962a6f6434361 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a54a22cf799142d9a9b962a6f6434361 2024-11-11T13:37:31,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/00f050ae46124af5a17bab7ee5fb93e4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/00f050ae46124af5a17bab7ee5fb93e4 2024-11-11T13:37:31,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/33a54e7034294175909cabe02a59043d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/33a54e7034294175909cabe02a59043d 2024-11-11T13:37:31,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1ce568dc4ac741e1b49f33481cc69232 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/1ce568dc4ac741e1b49f33481cc69232 2024-11-11T13:37:31,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/156c5483c2a4420d911157895a89b129 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/156c5483c2a4420d911157895a89b129 2024-11-11T13:37:31,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/ea9f7baa592f455781351c0c733e10ca to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/ea9f7baa592f455781351c0c733e10ca 2024-11-11T13:37:31,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/02bff4b5564f467c83db7762bd22515f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/02bff4b5564f467c83db7762bd22515f 2024-11-11T13:37:31,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5bb24b160a064955a8b616a38edf3a1a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5bb24b160a064955a8b616a38edf3a1a 2024-11-11T13:37:31,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b75bfa54e40d4ab09d3d8d6aece0d04e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b75bfa54e40d4ab09d3d8d6aece0d04e 2024-11-11T13:37:31,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/d7d060c50dbf49de938a593b4233830d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/d7d060c50dbf49de938a593b4233830d 2024-11-11T13:37:31,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/304536c32cf948aaaa314058d7290bfd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/304536c32cf948aaaa314058d7290bfd 2024-11-11T13:37:31,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/4747057b0a79423bbddf568fea03e93b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/4747057b0a79423bbddf568fea03e93b 2024-11-11T13:37:31,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92ce4cfe9bd44093be71e2aa5b57fa86 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92ce4cfe9bd44093be71e2aa5b57fa86 2024-11-11T13:37:31,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/69b7b4433ff04a94b90713d5f17b27c9 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/69b7b4433ff04a94b90713d5f17b27c9 2024-11-11T13:37:31,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/0ef00d8928cc45928f36cdb24635391f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/0ef00d8928cc45928f36cdb24635391f 2024-11-11T13:37:31,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/136e528a903d471c9bc681ec692a791f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/136e528a903d471c9bc681ec692a791f 2024-11-11T13:37:31,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92cb1f1828444a708a924a07ae8498bd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92cb1f1828444a708a924a07ae8498bd 2024-11-11T13:37:31,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92fc34e9d7714d5185de5e841e212bc1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92fc34e9d7714d5185de5e841e212bc1 2024-11-11T13:37:31,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/86d0018787c6447381ed4c03244c929f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/86d0018787c6447381ed4c03244c929f 2024-11-11T13:37:31,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/4e3773c93520431c97938363c2e274ad to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/4e3773c93520431c97938363c2e274ad 2024-11-11T13:37:31,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/acc29fab29c84b1bba483f57b146c39b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/acc29fab29c84b1bba483f57b146c39b 2024-11-11T13:37:31,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a466058f6e8f4b1880b78ef752c8fdd5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/a466058f6e8f4b1880b78ef752c8fdd5 2024-11-11T13:37:31,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5896cb69dd864b0bb6ae34b735942a71 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5896cb69dd864b0bb6ae34b735942a71 2024-11-11T13:37:31,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5a9ed34bd64b4c2f8d2031d3eca5a316 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/5a9ed34bd64b4c2f8d2031d3eca5a316 2024-11-11T13:37:31,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/182d8675f34f4301abb093de03efe116 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/182d8675f34f4301abb093de03efe116 2024-11-11T13:37:31,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/12ec9829f0244c9486bf9ca03b80411a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/12ec9829f0244c9486bf9ca03b80411a 2024-11-11T13:37:31,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b31f713df8564854a0636a447b132c95 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b31f713df8564854a0636a447b132c95 2024-11-11T13:37:31,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/e5c6c7b7be464928b8e070f4f55db916 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/e5c6c7b7be464928b8e070f4f55db916 2024-11-11T13:37:31,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b6297b3a7f0b4ce8ae3e1d5d99e36c54 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b6297b3a7f0b4ce8ae3e1d5d99e36c54 2024-11-11T13:37:31,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/46140763acdc436893769947a0d6b664 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/46140763acdc436893769947a0d6b664 2024-11-11T13:37:31,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/74e7325e67134cbda8679b0b6031eddc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/74e7325e67134cbda8679b0b6031eddc 2024-11-11T13:37:31,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/f2dca79f6739486a83c541a4897ef369 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/f2dca79f6739486a83c541a4897ef369 2024-11-11T13:37:31,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/bc9a3cd40ecb4997a6b19c35c1857702, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/604934cc2c9a464eb3871479dc4b5131, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8f24b5023eb7409bab408ad0928a61bd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a108c7697a724599a6511fabdbf6025a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/71d13b719ce34acca8f194134517934c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/cb041dba78ff45a684218dde553e55c0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/fb2ccf6d3cbb482dad3b6b0f96fc032a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8b7107b267914dc0bf460d0322c38135, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/52d2d8ab50644f13aaf3eea4581933d4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/c6b0fa434eb44197affcb58787777b92, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/f288f299bf1b4050a3a1d445437cc617, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a4907ad613f45c495d343b25bdafbf2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/f003b99ac9d642b8af232c3d756c04cd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a1ddd1b432542869836ecf2405c1899, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/e9b5f88199864a7f88566af05546574a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b35de2ea6200413da0299e2ba6cd3cd6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/9a331355e17c4fbba84c2b828b787c37, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/d37b5aa3416046f69b6b768dbc0f90b2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/aa1e580967704524a12b1fd12a70c4c9, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/47cfefca41344bb18f5694f42635bb7f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/2cbc816b155741e2bc10e4dd394320b6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/dd112f2f0a6840a7854bc4fae219719a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a2d544a87dac4b798ac2f6b8cb6e56ae, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/0285cc7eeb4f409586b924850abcbdf3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8e48fe112296485395c6fd2d99154181, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/ed18a0debea04b20a29a8dc5e40a2f2a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/66c9af7673bd424ba6cb0567b563f243, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a1062dd66e344c959064654b6ba3faf8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/83f57e34c2914ecfab16384f9e78b9c5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/38c432b1cd994aa4ba5faf87a60bf89c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/85474b8c799544a690344b7201b58a3b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/7342891b704448d4aa9fde2fc42f56e6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a4ed7a064799410aaee72c5e8e762936, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/4f752322d18f4b1988a0aa8df28c18c6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b02f1d421607486bad1921e254ee1abe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/2fb9468b16c74876b0d16462cacdfbb7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a3acdb1e91554090aee5426e510f2d52, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a4a0222bc4a148368ceddbd3d8b753ae, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b24ebc170c3e42ff94d2aa5000eb588e] to archive 2024-11-11T13:37:31,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:37:31,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/bc9a3cd40ecb4997a6b19c35c1857702 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/bc9a3cd40ecb4997a6b19c35c1857702 2024-11-11T13:37:31,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/604934cc2c9a464eb3871479dc4b5131 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/604934cc2c9a464eb3871479dc4b5131 2024-11-11T13:37:31,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8f24b5023eb7409bab408ad0928a61bd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8f24b5023eb7409bab408ad0928a61bd 2024-11-11T13:37:31,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a108c7697a724599a6511fabdbf6025a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a108c7697a724599a6511fabdbf6025a 2024-11-11T13:37:31,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/71d13b719ce34acca8f194134517934c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/71d13b719ce34acca8f194134517934c 2024-11-11T13:37:31,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/cb041dba78ff45a684218dde553e55c0 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/cb041dba78ff45a684218dde553e55c0 2024-11-11T13:37:31,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/fb2ccf6d3cbb482dad3b6b0f96fc032a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/fb2ccf6d3cbb482dad3b6b0f96fc032a 2024-11-11T13:37:31,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8b7107b267914dc0bf460d0322c38135 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8b7107b267914dc0bf460d0322c38135 2024-11-11T13:37:31,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/52d2d8ab50644f13aaf3eea4581933d4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/52d2d8ab50644f13aaf3eea4581933d4 2024-11-11T13:37:31,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/c6b0fa434eb44197affcb58787777b92 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/c6b0fa434eb44197affcb58787777b92 2024-11-11T13:37:31,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/f288f299bf1b4050a3a1d445437cc617 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/f288f299bf1b4050a3a1d445437cc617 2024-11-11T13:37:31,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a4907ad613f45c495d343b25bdafbf2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a4907ad613f45c495d343b25bdafbf2 2024-11-11T13:37:31,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/f003b99ac9d642b8af232c3d756c04cd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/f003b99ac9d642b8af232c3d756c04cd 2024-11-11T13:37:31,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a1ddd1b432542869836ecf2405c1899 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1a1ddd1b432542869836ecf2405c1899 2024-11-11T13:37:31,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/e9b5f88199864a7f88566af05546574a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/e9b5f88199864a7f88566af05546574a 2024-11-11T13:37:31,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b35de2ea6200413da0299e2ba6cd3cd6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b35de2ea6200413da0299e2ba6cd3cd6 2024-11-11T13:37:31,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/9a331355e17c4fbba84c2b828b787c37 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/9a331355e17c4fbba84c2b828b787c37 2024-11-11T13:37:31,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/d37b5aa3416046f69b6b768dbc0f90b2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/d37b5aa3416046f69b6b768dbc0f90b2 2024-11-11T13:37:31,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/aa1e580967704524a12b1fd12a70c4c9 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/aa1e580967704524a12b1fd12a70c4c9 2024-11-11T13:37:31,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/47cfefca41344bb18f5694f42635bb7f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/47cfefca41344bb18f5694f42635bb7f 2024-11-11T13:37:31,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/2cbc816b155741e2bc10e4dd394320b6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/2cbc816b155741e2bc10e4dd394320b6 2024-11-11T13:37:31,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/dd112f2f0a6840a7854bc4fae219719a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/dd112f2f0a6840a7854bc4fae219719a 2024-11-11T13:37:31,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a2d544a87dac4b798ac2f6b8cb6e56ae to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a2d544a87dac4b798ac2f6b8cb6e56ae 2024-11-11T13:37:31,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/0285cc7eeb4f409586b924850abcbdf3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/0285cc7eeb4f409586b924850abcbdf3 2024-11-11T13:37:31,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8e48fe112296485395c6fd2d99154181 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8e48fe112296485395c6fd2d99154181 2024-11-11T13:37:31,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/ed18a0debea04b20a29a8dc5e40a2f2a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/ed18a0debea04b20a29a8dc5e40a2f2a 2024-11-11T13:37:31,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/66c9af7673bd424ba6cb0567b563f243 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/66c9af7673bd424ba6cb0567b563f243 2024-11-11T13:37:31,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a1062dd66e344c959064654b6ba3faf8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a1062dd66e344c959064654b6ba3faf8 2024-11-11T13:37:31,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/83f57e34c2914ecfab16384f9e78b9c5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/83f57e34c2914ecfab16384f9e78b9c5 2024-11-11T13:37:31,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/38c432b1cd994aa4ba5faf87a60bf89c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/38c432b1cd994aa4ba5faf87a60bf89c 2024-11-11T13:37:31,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/85474b8c799544a690344b7201b58a3b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/85474b8c799544a690344b7201b58a3b 2024-11-11T13:37:31,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/7342891b704448d4aa9fde2fc42f56e6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/7342891b704448d4aa9fde2fc42f56e6 2024-11-11T13:37:31,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a4ed7a064799410aaee72c5e8e762936 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a4ed7a064799410aaee72c5e8e762936 2024-11-11T13:37:31,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/4f752322d18f4b1988a0aa8df28c18c6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/4f752322d18f4b1988a0aa8df28c18c6 2024-11-11T13:37:31,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b02f1d421607486bad1921e254ee1abe to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b02f1d421607486bad1921e254ee1abe 2024-11-11T13:37:31,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/2fb9468b16c74876b0d16462cacdfbb7 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/2fb9468b16c74876b0d16462cacdfbb7 2024-11-11T13:37:31,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a3acdb1e91554090aee5426e510f2d52 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a3acdb1e91554090aee5426e510f2d52 2024-11-11T13:37:31,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a4a0222bc4a148368ceddbd3d8b753ae to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/a4a0222bc4a148368ceddbd3d8b753ae 2024-11-11T13:37:31,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b24ebc170c3e42ff94d2aa5000eb588e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b24ebc170c3e42ff94d2aa5000eb588e 2024-11-11T13:37:31,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49566b23b50d491896454693f0a55fee, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/a2f5603dbe9348e1bac007c61ddb133a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/3d1465fe134e48eaa56d49921d3e67a3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49e5ec4b34b04e8f9163c8966746c7e2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/e048c3329bd546c08234f8d769781238, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/8e5e3ee7c64c4cd2ba75ba1a0db92644, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/581824db58c7422890498cabf7d848a0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4d191381509746659a21d466d92777c7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/24983221302e42a4b0286687a4755a33, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/9275fe30ebab4a4f8bca968f1999145c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/bc87bd7701224b36b3c8e80f8c4b036f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2616e717d574488e833867dccde7df0f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/0a6665b744f84a799327882bf110a07d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2dd3a1e76fa14a45bf433a1e50e8bc6c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/db4dc401f46f41d3ab9e00331514346f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/6d61a344242b42608fd4e7227c3ef6d7, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/b5e57f9370db4fcc8e53856b4b03aeaf, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/0b1b2b11eff34af49ee8dfe0235afcf1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4a42843ccb1a4444be8a01a1de67d330, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4b8e5113dbfc4d06a10efa7eb0fcea38, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/48af5f55e7474f70884484ca31f07c13, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/aa78ce59c21b4796ba63ff06dc8751cd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/158967b15f314f95a12b9596dac6f484, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f62d493aac044044aa22c898a1286662, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/8b89a19a6ff044c4a30186e99a4ca4d0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/c9840ea42a80421ab4beff98e8a40d40, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/267736de5db94bb99f730afb41c881f3, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f31a9a4d5c7c412d8cd9f84ec8223afd, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/dba12da380294ef4b1e4345679b620d0, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/5769ab21b45f434d995486ab53c44ab8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/b4ec693a35e3468281f42630e9212fec, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f898484e189c43fab1c24d36e4f7b715, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/1b9ed88af06e421dabb1de49f94e702f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/5eefd1e0681e484b801647342795e351, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/28381b1ec9464fb4a6be18c5e407dcab, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/6a3d8f955ba44a9fb4b5cf7b5b6d9592, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/76e5c861f4b340b79ae37429731a9587, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/11e7a409e2cf4da6802e0aa5a3a5f0d1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f8830314989a44798e4caa51eeabea31] to archive 2024-11-11T13:37:31,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:37:31,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49566b23b50d491896454693f0a55fee to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49566b23b50d491896454693f0a55fee 2024-11-11T13:37:31,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/a2f5603dbe9348e1bac007c61ddb133a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/a2f5603dbe9348e1bac007c61ddb133a 2024-11-11T13:37:31,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/3d1465fe134e48eaa56d49921d3e67a3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/3d1465fe134e48eaa56d49921d3e67a3 2024-11-11T13:37:31,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49e5ec4b34b04e8f9163c8966746c7e2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/49e5ec4b34b04e8f9163c8966746c7e2 2024-11-11T13:37:31,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/e048c3329bd546c08234f8d769781238 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/e048c3329bd546c08234f8d769781238 2024-11-11T13:37:31,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/8e5e3ee7c64c4cd2ba75ba1a0db92644 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/8e5e3ee7c64c4cd2ba75ba1a0db92644 2024-11-11T13:37:31,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/581824db58c7422890498cabf7d848a0 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/581824db58c7422890498cabf7d848a0 2024-11-11T13:37:31,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4d191381509746659a21d466d92777c7 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4d191381509746659a21d466d92777c7 2024-11-11T13:37:31,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/24983221302e42a4b0286687a4755a33 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/24983221302e42a4b0286687a4755a33 2024-11-11T13:37:31,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/9275fe30ebab4a4f8bca968f1999145c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/9275fe30ebab4a4f8bca968f1999145c 2024-11-11T13:37:31,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/bc87bd7701224b36b3c8e80f8c4b036f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/bc87bd7701224b36b3c8e80f8c4b036f 2024-11-11T13:37:31,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2616e717d574488e833867dccde7df0f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2616e717d574488e833867dccde7df0f 2024-11-11T13:37:31,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/0a6665b744f84a799327882bf110a07d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/0a6665b744f84a799327882bf110a07d 2024-11-11T13:37:31,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2dd3a1e76fa14a45bf433a1e50e8bc6c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/2dd3a1e76fa14a45bf433a1e50e8bc6c 2024-11-11T13:37:31,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/db4dc401f46f41d3ab9e00331514346f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/db4dc401f46f41d3ab9e00331514346f 2024-11-11T13:37:31,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/6d61a344242b42608fd4e7227c3ef6d7 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/6d61a344242b42608fd4e7227c3ef6d7 2024-11-11T13:37:31,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/b5e57f9370db4fcc8e53856b4b03aeaf to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/b5e57f9370db4fcc8e53856b4b03aeaf 2024-11-11T13:37:31,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/0b1b2b11eff34af49ee8dfe0235afcf1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/0b1b2b11eff34af49ee8dfe0235afcf1 2024-11-11T13:37:31,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4a42843ccb1a4444be8a01a1de67d330 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4a42843ccb1a4444be8a01a1de67d330 2024-11-11T13:37:31,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4b8e5113dbfc4d06a10efa7eb0fcea38 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/4b8e5113dbfc4d06a10efa7eb0fcea38 2024-11-11T13:37:31,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/48af5f55e7474f70884484ca31f07c13 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/48af5f55e7474f70884484ca31f07c13 2024-11-11T13:37:31,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/aa78ce59c21b4796ba63ff06dc8751cd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/aa78ce59c21b4796ba63ff06dc8751cd 2024-11-11T13:37:31,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/158967b15f314f95a12b9596dac6f484 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/158967b15f314f95a12b9596dac6f484 2024-11-11T13:37:31,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f62d493aac044044aa22c898a1286662 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f62d493aac044044aa22c898a1286662 2024-11-11T13:37:31,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/8b89a19a6ff044c4a30186e99a4ca4d0 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/8b89a19a6ff044c4a30186e99a4ca4d0 2024-11-11T13:37:31,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/c9840ea42a80421ab4beff98e8a40d40 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/c9840ea42a80421ab4beff98e8a40d40 2024-11-11T13:37:31,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/267736de5db94bb99f730afb41c881f3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/267736de5db94bb99f730afb41c881f3 2024-11-11T13:37:31,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f31a9a4d5c7c412d8cd9f84ec8223afd to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f31a9a4d5c7c412d8cd9f84ec8223afd 2024-11-11T13:37:31,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/dba12da380294ef4b1e4345679b620d0 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/dba12da380294ef4b1e4345679b620d0 2024-11-11T13:37:31,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/5769ab21b45f434d995486ab53c44ab8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/5769ab21b45f434d995486ab53c44ab8 2024-11-11T13:37:31,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/b4ec693a35e3468281f42630e9212fec to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/b4ec693a35e3468281f42630e9212fec 2024-11-11T13:37:31,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f898484e189c43fab1c24d36e4f7b715 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f898484e189c43fab1c24d36e4f7b715 2024-11-11T13:37:31,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/1b9ed88af06e421dabb1de49f94e702f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/1b9ed88af06e421dabb1de49f94e702f 2024-11-11T13:37:31,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/5eefd1e0681e484b801647342795e351 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/5eefd1e0681e484b801647342795e351 2024-11-11T13:37:31,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/28381b1ec9464fb4a6be18c5e407dcab to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/28381b1ec9464fb4a6be18c5e407dcab 2024-11-11T13:37:31,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/6a3d8f955ba44a9fb4b5cf7b5b6d9592 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/6a3d8f955ba44a9fb4b5cf7b5b6d9592 2024-11-11T13:37:31,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/76e5c861f4b340b79ae37429731a9587 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/76e5c861f4b340b79ae37429731a9587 2024-11-11T13:37:31,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/11e7a409e2cf4da6802e0aa5a3a5f0d1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/11e7a409e2cf4da6802e0aa5a3a5f0d1 2024-11-11T13:37:31,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f8830314989a44798e4caa51eeabea31 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/f8830314989a44798e4caa51eeabea31 2024-11-11T13:37:31,109 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/recovered.edits/574.seqid, newMaxSeqId=574, maxSeqId=1 2024-11-11T13:37:31,110 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. 2024-11-11T13:37:31,110 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for ed96da846868b1163b5c13dff6f86179: Waiting for close lock at 1731332249763Running coprocessor pre-close hooks at 1731332249763Disabling compacts and flushes for region at 1731332249763Disabling writes for close at 1731332249763Obtaining lock to block concurrent updates at 1731332249763Preparing flush snapshotting stores in ed96da846868b1163b5c13dff6f86179 at 1731332249763Finished memstore snapshotting TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731332249763Flushing stores of TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179. at 1731332249764 (+1 ms)Flushing ed96da846868b1163b5c13dff6f86179/A: creating writer at 1731332249764Flushing ed96da846868b1163b5c13dff6f86179/A: appending metadata at 1731332249766 (+2 ms)Flushing ed96da846868b1163b5c13dff6f86179/A: closing flushed file at 1731332249766Flushing ed96da846868b1163b5c13dff6f86179/B: creating writer at 1731332250180 (+414 ms)Flushing ed96da846868b1163b5c13dff6f86179/B: appending metadata at 1731332250182 (+2 ms)Flushing ed96da846868b1163b5c13dff6f86179/B: closing flushed file at 1731332250182Flushing ed96da846868b1163b5c13dff6f86179/C: creating writer at 1731332250589 (+407 ms)Flushing ed96da846868b1163b5c13dff6f86179/C: appending metadata at 1731332250592 (+3 ms)Flushing ed96da846868b1163b5c13dff6f86179/C: closing flushed file at 1731332250592Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cc8b1e9: reopening flushed file at 1731332251003 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6798d150: reopening flushed file at 1731332251009 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@434ef228: reopening flushed file at 1731332251013 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for ed96da846868b1163b5c13dff6f86179 in 1254ms, sequenceid=571, compaction requested=true at 1731332251018 (+5 ms)Writing region close event to WAL at 1731332251107 (+89 ms)Running coprocessor post-close hooks at 1731332251110 (+3 ms)Closed at 1731332251110 2024-11-11T13:37:31,112 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:31,112 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=ed96da846868b1163b5c13dff6f86179, regionState=CLOSED 2024-11-11T13:37:31,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:37:31,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T13:37:31,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure ed96da846868b1163b5c13dff6f86179, server=37d26ecf6460,36877,1731332101576 in 1.5040 sec 2024-11-11T13:37:31,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-11T13:37:31,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ed96da846868b1163b5c13dff6f86179, UNASSIGN in 1.5080 sec 2024-11-11T13:37:31,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T13:37:31,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5110 sec 2024-11-11T13:37:31,121 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332251121"}]},"ts":"1731332251121"} 2024-11-11T13:37:31,122 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T13:37:31,123 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T13:37:31,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5200 sec 2024-11-11T13:37:31,161 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T13:37:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T13:37:31,750 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:31,750 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T13:37:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:31,752 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T13:37:31,753 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:31,754 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:31,756 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/recovered.edits] 2024-11-11T13:37:31,758 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92228bfcca5d4663857241353325a784 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/92228bfcca5d4663857241353325a784 2024-11-11T13:37:31,759 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b72ea696509d4347ab4ce26e62d53a15 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/b72ea696509d4347ab4ce26e62d53a15 2024-11-11T13:37:31,760 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/bceb09b359b34559a834fc1b967bd145 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/A/bceb09b359b34559a834fc1b967bd145 2024-11-11T13:37:31,762 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1c1b751bedec487a8159d47ba4942760 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/1c1b751bedec487a8159d47ba4942760 2024-11-11T13:37:31,763 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8819df35cbf14ecabdc4f3f2eb45f497 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/8819df35cbf14ecabdc4f3f2eb45f497 2024-11-11T13:37:31,764 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b2ec13172b134dc5aee2e36cec89f49f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/B/b2ec13172b134dc5aee2e36cec89f49f 2024-11-11T13:37:31,765 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/06700996dada44ffbcec03c0306101f3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/06700996dada44ffbcec03c0306101f3 2024-11-11T13:37:31,765 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/1a780cf804b74064a25dfa74a8c74ff2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/1a780cf804b74064a25dfa74a8c74ff2 2024-11-11T13:37:31,766 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/3489aa78f4ef4cb99766a052b4c8b34d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/C/3489aa78f4ef4cb99766a052b4c8b34d 2024-11-11T13:37:31,768 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/recovered.edits/574.seqid to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179/recovered.edits/574.seqid 2024-11-11T13:37:31,768 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/ed96da846868b1163b5c13dff6f86179 2024-11-11T13:37:31,768 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T13:37:31,770 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:31,771 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T13:37:31,773 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T13:37:31,774 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:31,774 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T13:37:31,774 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731332251774"}]},"ts":"9223372036854775807"} 2024-11-11T13:37:31,775 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T13:37:31,775 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ed96da846868b1163b5c13dff6f86179, NAME => 'TestAcidGuarantees,,1731332222446.ed96da846868b1163b5c13dff6f86179.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T13:37:31,775 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T13:37:31,776 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731332251775"}]},"ts":"9223372036854775807"} 2024-11-11T13:37:31,777 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T13:37:31,777 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:31,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-11-11T13:37:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T13:37:31,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T13:37:31,862 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:31,868 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239 (was 241), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=662 (was 577) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2594 (was 2738) 2024-11-11T13:37:31,875 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=662, ProcessCount=11, AvailableMemoryMB=2588 2024-11-11T13:37:31,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T13:37:31,876 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:37:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:31,878 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T13:37:31,878 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:31,878 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-11T13:37:31,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T13:37:31,879 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T13:37:31,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742373_1549 (size=959) 2024-11-11T13:37:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T13:37:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T13:37:32,285 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92 2024-11-11T13:37:32,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742374_1550 (size=53) 2024-11-11T13:37:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T13:37:32,691 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:37:32,691 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f177ccda479452e8555a646c67251509, disabling compactions & flushes 2024-11-11T13:37:32,691 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:32,691 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:32,691 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. after waiting 0 ms 2024-11-11T13:37:32,691 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:32,691 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:32,691 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f177ccda479452e8555a646c67251509: Waiting for close lock at 1731332252691Disabling compacts and flushes for region at 1731332252691Disabling writes for close at 1731332252691Writing region close event to WAL at 1731332252691Closed at 1731332252691 2024-11-11T13:37:32,692 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T13:37:32,692 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731332252692"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731332252692"}]},"ts":"1731332252692"} 2024-11-11T13:37:32,693 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T13:37:32,694 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T13:37:32,694 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332252694"}]},"ts":"1731332252694"} 2024-11-11T13:37:32,695 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T13:37:32,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, ASSIGN}] 2024-11-11T13:37:32,696 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, ASSIGN 2024-11-11T13:37:32,697 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, ASSIGN; state=OFFLINE, location=37d26ecf6460,36877,1731332101576; forceNewPlan=false, retain=false 2024-11-11T13:37:32,847 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=f177ccda479452e8555a646c67251509, regionState=OPENING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:32,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, ASSIGN because future has completed 2024-11-11T13:37:32,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:37:33,003 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:33,003 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:37:33,003 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,004 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:37:33,004 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,004 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,005 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,006 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f177ccda479452e8555a646c67251509 columnFamilyName A 2024-11-11T13:37:33,006 DEBUG [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:33,007 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(327): Store=f177ccda479452e8555a646c67251509/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:37:33,007 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,008 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f177ccda479452e8555a646c67251509 columnFamilyName B 2024-11-11T13:37:33,008 DEBUG [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:33,008 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(327): Store=f177ccda479452e8555a646c67251509/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:37:33,008 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,010 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f177ccda479452e8555a646c67251509 columnFamilyName C 2024-11-11T13:37:33,010 DEBUG [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:33,010 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(327): Store=f177ccda479452e8555a646c67251509/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:37:33,010 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,011 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,011 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,012 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,012 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,012 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T13:37:33,013 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,015 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:37:33,015 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened f177ccda479452e8555a646c67251509; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61750596, jitterRate=-0.0798444151878357}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T13:37:33,015 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f177ccda479452e8555a646c67251509 2024-11-11T13:37:33,016 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for f177ccda479452e8555a646c67251509: Running coprocessor pre-open hook at 1731332253004Writing region info on filesystem at 1731332253004Initializing all the Stores at 1731332253004Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332253004Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332253005 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332253005Cleaning up temporary data from old regions at 1731332253012 (+7 ms)Running coprocessor post-open hooks at 1731332253015 (+3 ms)Region opened successfully at 1731332253016 (+1 ms) 2024-11-11T13:37:33,016 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., pid=138, masterSystemTime=1731332253000 2024-11-11T13:37:33,018 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:33,018 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T13:37:33,019 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=f177ccda479452e8555a646c67251509, regionState=OPEN, openSeqNum=2, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:33,020 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:37:33,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-11T13:37:33,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 in 172 msec 2024-11-11T13:37:33,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-11T13:37:33,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, ASSIGN in 327 msec 2024-11-11T13:37:33,024 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T13:37:33,025 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332253024"}]},"ts":"1731332253024"} 2024-11-11T13:37:33,026 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T13:37:33,026 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T13:37:33,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-11T13:37:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T13:37:34,029 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:34,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef845b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:34,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:34,031 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac6ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:34,033 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1fdd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,034 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:34,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:34,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:34,036 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T13:37:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 37d26ecf6460,35609,1731332100707 2024-11-11T13:37:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3fcab6cd 2024-11-11T13:37:34,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T13:37:34,038 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T13:37:34,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T13:37:34,040 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:37:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:34,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742375_1551 (size=995) 2024-11-11T13:37:34,454 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-11T13:37:34,454 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-11T13:37:34,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T13:37:34,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, REOPEN/MOVE}] 2024-11-11T13:37:34,460 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, REOPEN/MOVE 2024-11-11T13:37:34,460 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=f177ccda479452e8555a646c67251509, regionState=CLOSING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:34,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, REOPEN/MOVE because future has completed 2024-11-11T13:37:34,462 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T13:37:34,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:37:34,614 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,614 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T13:37:34,614 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing f177ccda479452e8555a646c67251509, disabling compactions & flushes 2024-11-11T13:37:34,614 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:34,614 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:34,614 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. after waiting 0 ms 2024-11-11T13:37:34,614 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:34,617 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T13:37:34,617 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:34,617 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for f177ccda479452e8555a646c67251509: Waiting for close lock at 1731332254614Running coprocessor pre-close hooks at 1731332254614Disabling compacts and flushes for region at 1731332254614Disabling writes for close at 1731332254614Writing region close event to WAL at 1731332254614Running coprocessor post-close hooks at 1731332254617 (+3 ms)Closed at 1731332254617 2024-11-11T13:37:34,617 WARN [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: f177ccda479452e8555a646c67251509 to self. 2024-11-11T13:37:34,619 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,619 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=f177ccda479452e8555a646c67251509, regionState=CLOSED 2024-11-11T13:37:34,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:37:34,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-11T13:37:34,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 in 159 msec 2024-11-11T13:37:34,623 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, REOPEN/MOVE; state=CLOSED, location=37d26ecf6460,36877,1731332101576; forceNewPlan=false, retain=true 2024-11-11T13:37:34,773 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=f177ccda479452e8555a646c67251509, regionState=OPENING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:34,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, REOPEN/MOVE because future has completed 2024-11-11T13:37:34,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:37:34,930 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:34,930 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:37:34,930 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,930 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:37:34,930 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,930 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,931 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,932 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f177ccda479452e8555a646c67251509 columnFamilyName A 2024-11-11T13:37:34,932 DEBUG [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:34,933 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(327): Store=f177ccda479452e8555a646c67251509/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:37:34,933 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,934 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f177ccda479452e8555a646c67251509 columnFamilyName B 2024-11-11T13:37:34,934 DEBUG [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:34,934 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(327): Store=f177ccda479452e8555a646c67251509/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:37:34,934 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,935 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f177ccda479452e8555a646c67251509 columnFamilyName C 2024-11-11T13:37:34,935 DEBUG [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:34,935 INFO [StoreOpener-f177ccda479452e8555a646c67251509-1 {}] regionserver.HStore(327): Store=f177ccda479452e8555a646c67251509/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:37:34,935 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,936 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,936 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,937 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,937 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,937 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T13:37:34,938 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,939 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened f177ccda479452e8555a646c67251509; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62552061, jitterRate=-0.06790165603160858}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T13:37:34,939 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f177ccda479452e8555a646c67251509 2024-11-11T13:37:34,939 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for f177ccda479452e8555a646c67251509: Running coprocessor pre-open hook at 1731332254930Writing region info on filesystem at 1731332254930Initializing all the Stores at 1731332254931 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332254931Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332254931Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731332254931Cleaning up temporary data from old regions at 1731332254937 (+6 ms)Running coprocessor post-open hooks at 1731332254939 (+2 ms)Region opened successfully at 1731332254939 2024-11-11T13:37:34,940 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., pid=143, masterSystemTime=1731332254927 2024-11-11T13:37:34,941 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:34,941 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:34,942 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=f177ccda479452e8555a646c67251509, regionState=OPEN, openSeqNum=5, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:34,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:37:34,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-11T13:37:34,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 in 169 msec 2024-11-11T13:37:34,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-11T13:37:34,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, REOPEN/MOVE in 486 msec 2024-11-11T13:37:34,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T13:37:34,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-11T13:37:34,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-11-11T13:37:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T13:37:34,951 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:34,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cac35db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:34,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:34,952 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:34,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:34,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:34,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fc2d6a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:34,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7254cb75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df37b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,954 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:34,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:34,955 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34afe5c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:34,956 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6e7732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40067e81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:34,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:34,957 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3950b5c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:34,958 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43de2dba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@116f9018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:34,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:34,959 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cc92490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:34,961 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c0c8954, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4e69c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,962 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:34,962 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:34,963 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ee66f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:34,964 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:34,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4399b5d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fe30fa0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:34,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:34,967 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:34,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:34,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:34,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61bcfe5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:34,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:34,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:34,968 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:34,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227bddae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c13258c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:34,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:34,971 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f57d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:34,972 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b53700b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703fbe97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:34,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:34,974 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:34,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@229692db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:34,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:34,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:34,976 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:34,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9fac68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dfe678f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:34,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:34,977 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@202976e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:34,979 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320fb8d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f84826b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,35609,-1 for getting cluster id 2024-11-11T13:37:34,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:37:34,981 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7f38533c-3e78-4919-855f-edcf552b3d40' 2024-11-11T13:37:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:37:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7f38533c-3e78-4919-855f-edcf552b3d40" 2024-11-11T13:37:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ea3fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,35609,-1] 2024-11-11T13:37:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:37:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:34,982 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:37:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34bb5b04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:37:34,983 DEBUG [Thread-2471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:34,983 DEBUG [Thread-2460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:34,987 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:34,987 DEBUG [Thread-2458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:34,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:34,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:37:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:37:34,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:34,992 DEBUG [Thread-2465 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:37:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:34,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:34,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:37:35,001 DEBUG [Thread-2467 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:35,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:37:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:35,007 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:35,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:37:35,014 DEBUG [Thread-2469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:35,022 DEBUG [Thread-2463 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:35,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:35,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:35,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:35,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:35,025 DEBUG [Thread-2456 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:37:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:37:35,030 DEBUG [Thread-2454 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:35,030 DEBUG [Thread-2452 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:37:35,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:35,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:35,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,36877,1731332101576, seqNum=-1] 2024-11-11T13:37:35,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:35,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:35,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:37:35,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:35,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:35,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:37:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:37:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:37:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5] 2024-11-11T13:37:35,049 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-11T13:37:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T13:37:35,055 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:35,056 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:35,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:35,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332315055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332315056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332315057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332315061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332315062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111170f174de9b0949aa9695eb26529c4841_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332255007/Put/seqid=0 2024-11-11T13:37:35,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742376_1552 (size=12154) 2024-11-11T13:37:35,112 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:35,127 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111170f174de9b0949aa9695eb26529c4841_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170f174de9b0949aa9695eb26529c4841_f177ccda479452e8555a646c67251509 2024-11-11T13:37:35,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/ee57bb59dc8946ad950aa4378062e17d, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:35,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/ee57bb59dc8946ad950aa4378062e17d is 175, key is test_row_0/A:col10/1731332255007/Put/seqid=0 2024-11-11T13:37:35,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742377_1553 (size=30955) 2024-11-11T13:37:35,135 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/ee57bb59dc8946ad950aa4378062e17d 2024-11-11T13:37:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T13:37:35,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332315172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332315172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332315172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332315172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332315172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/227a7dcaaf704c7881ad19079a461be2 is 50, key is test_row_0/B:col10/1731332255007/Put/seqid=0 2024-11-11T13:37:35,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T13:37:35,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:35,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742378_1554 (size=12001) 2024-11-11T13:37:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T13:37:35,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T13:37:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332315379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332315379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332315380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332315382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332315382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T13:37:35,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:35,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/227a7dcaaf704c7881ad19079a461be2 2024-11-11T13:37:35,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/c28d12eaab144924b3866d192817ee45 is 50, key is test_row_0/C:col10/1731332255007/Put/seqid=0 2024-11-11T13:37:35,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742379_1555 (size=12001) 2024-11-11T13:37:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T13:37:35,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T13:37:35,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:35,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332315689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332315690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332315690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332315690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332315690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:35,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T13:37:35,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:35,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T13:37:35,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:35,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:35,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:35,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:36,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/c28d12eaab144924b3866d192817ee45 2024-11-11T13:37:36,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/ee57bb59dc8946ad950aa4378062e17d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ee57bb59dc8946ad950aa4378062e17d 2024-11-11T13:37:36,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ee57bb59dc8946ad950aa4378062e17d, entries=150, sequenceid=16, filesize=30.2 K 2024-11-11T13:37:36,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/227a7dcaaf704c7881ad19079a461be2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/227a7dcaaf704c7881ad19079a461be2 2024-11-11T13:37:36,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/227a7dcaaf704c7881ad19079a461be2, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T13:37:36,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/c28d12eaab144924b3866d192817ee45 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c28d12eaab144924b3866d192817ee45 2024-11-11T13:37:36,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c28d12eaab144924b3866d192817ee45, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T13:37:36,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f177ccda479452e8555a646c67251509 in 1077ms, sequenceid=16, compaction requested=false 2024-11-11T13:37:36,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:36,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T13:37:36,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:36,155 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T13:37:36,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bf7ab6c91b3540949e0eaee306b5cacc_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332255055/Put/seqid=0 2024-11-11T13:37:36,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742380_1556 (size=12154) 2024-11-11T13:37:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T13:37:36,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:36,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332316208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332316209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,216 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332316215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332316217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332316218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332316320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332316320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332316330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332316330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332316331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332316530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332316530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332316541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332316541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332316541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:36,590 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bf7ab6c91b3540949e0eaee306b5cacc_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bf7ab6c91b3540949e0eaee306b5cacc_f177ccda479452e8555a646c67251509 2024-11-11T13:37:36,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b380c4eb34f644c981a0d8db7b6acd30, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:36,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b380c4eb34f644c981a0d8db7b6acd30 is 175, key is test_row_0/A:col10/1731332255055/Put/seqid=0 2024-11-11T13:37:36,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742381_1557 (size=30955) 2024-11-11T13:37:36,617 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b380c4eb34f644c981a0d8db7b6acd30 2024-11-11T13:37:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/21535d11bbbb4f5ba09d642fbe8527cb is 50, key is test_row_0/B:col10/1731332255055/Put/seqid=0 2024-11-11T13:37:36,634 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T13:37:36,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742382_1558 (size=12001) 2024-11-11T13:37:36,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332316840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332316840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332316849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332316849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332316849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,043 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/21535d11bbbb4f5ba09d642fbe8527cb 2024-11-11T13:37:37,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/37eda10f8a6845e8809272e397ce4238 is 50, key is test_row_0/C:col10/1731332255055/Put/seqid=0 2024-11-11T13:37:37,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742383_1559 (size=12001) 2024-11-11T13:37:37,063 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/37eda10f8a6845e8809272e397ce4238 2024-11-11T13:37:37,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b380c4eb34f644c981a0d8db7b6acd30 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b380c4eb34f644c981a0d8db7b6acd30 2024-11-11T13:37:37,078 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b380c4eb34f644c981a0d8db7b6acd30, entries=150, sequenceid=40, filesize=30.2 K 2024-11-11T13:37:37,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/21535d11bbbb4f5ba09d642fbe8527cb as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/21535d11bbbb4f5ba09d642fbe8527cb 2024-11-11T13:37:37,094 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/21535d11bbbb4f5ba09d642fbe8527cb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T13:37:37,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/37eda10f8a6845e8809272e397ce4238 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/37eda10f8a6845e8809272e397ce4238 2024-11-11T13:37:37,110 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/37eda10f8a6845e8809272e397ce4238, entries=150, sequenceid=40, filesize=11.7 K 2024-11-11T13:37:37,111 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for f177ccda479452e8555a646c67251509 in 956ms, sequenceid=40, compaction requested=false 2024-11-11T13:37:37,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:37,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:37,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-11T13:37:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-11T13:37:37,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-11T13:37:37,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0620 sec 2024-11-11T13:37:37,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.0780 sec 2024-11-11T13:37:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T13:37:37,199 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:37,200 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-11T13:37:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T13:37:37,212 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:37,215 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:37,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T13:37:37,351 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:37,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T13:37:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:37,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T13:37:37,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:37,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:37,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:37,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b55d1bbb8e754dfbb19b290d607d677f_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332256213/Put/seqid=0 2024-11-11T13:37:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742384_1560 (size=12154) 2024-11-11T13:37:37,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332317382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332317382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332317384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332317384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332317384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,395 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332317499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332317499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332317499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332317499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332317499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T13:37:37,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T13:37:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:37,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T13:37:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:37,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332317710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332317710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332317710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332317710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332317710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:37,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:37,824 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b55d1bbb8e754dfbb19b290d607d677f_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b55d1bbb8e754dfbb19b290d607d677f_f177ccda479452e8555a646c67251509 2024-11-11T13:37:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T13:37:37,836 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/18dee53d48ee47e8baa49e3d6d90fbee, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:37,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/18dee53d48ee47e8baa49e3d6d90fbee is 175, key is test_row_0/A:col10/1731332256213/Put/seqid=0 2024-11-11T13:37:37,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T13:37:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:37,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:37,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742385_1561 (size=30955) 2024-11-11T13:37:37,856 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/18dee53d48ee47e8baa49e3d6d90fbee 2024-11-11T13:37:37,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/c39fdd791fbd450896b03640951888b4 is 50, key is test_row_0/B:col10/1731332256213/Put/seqid=0 2024-11-11T13:37:37,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742386_1562 (size=12001) 2024-11-11T13:37:38,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T13:37:38,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:38,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:38,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:38,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:38,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332318019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332318020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332318020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332318020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332318029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T13:37:38,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:38,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:38,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:38,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:38,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:38,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/c39fdd791fbd450896b03640951888b4 2024-11-11T13:37:38,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/d93a64744f394b12bdac2995d28746b5 is 50, key is test_row_0/C:col10/1731332256213/Put/seqid=0 2024-11-11T13:37:38,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T13:37:38,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:38,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:38,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:38,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:38,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:38,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:38,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742387_1563 (size=12001) 2024-11-11T13:37:38,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/d93a64744f394b12bdac2995d28746b5 2024-11-11T13:37:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T13:37:38,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/18dee53d48ee47e8baa49e3d6d90fbee as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/18dee53d48ee47e8baa49e3d6d90fbee 2024-11-11T13:37:38,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/18dee53d48ee47e8baa49e3d6d90fbee, entries=150, sequenceid=54, filesize=30.2 K 2024-11-11T13:37:38,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/c39fdd791fbd450896b03640951888b4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/c39fdd791fbd450896b03640951888b4 2024-11-11T13:37:38,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/c39fdd791fbd450896b03640951888b4, entries=150, sequenceid=54, filesize=11.7 K 2024-11-11T13:37:38,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/d93a64744f394b12bdac2995d28746b5 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/d93a64744f394b12bdac2995d28746b5 2024-11-11T13:37:38,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/d93a64744f394b12bdac2995d28746b5, entries=150, sequenceid=54, filesize=11.7 K 2024-11-11T13:37:38,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f177ccda479452e8555a646c67251509 in 1086ms, sequenceid=54, compaction requested=true 2024-11-11T13:37:38,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:38,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:38,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:38,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:38,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:37:38,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:38,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T13:37:38,438 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:38,438 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:38,440 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:38,440 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/C is initiating minor compaction (all files) 2024-11-11T13:37:38,440 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/C in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:38,440 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c28d12eaab144924b3866d192817ee45, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/37eda10f8a6845e8809272e397ce4238, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/d93a64744f394b12bdac2995d28746b5] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=35.2 K 2024-11-11T13:37:38,440 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:38,440 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/A is initiating minor compaction (all files) 2024-11-11T13:37:38,440 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/A in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:38,440 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ee57bb59dc8946ad950aa4378062e17d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b380c4eb34f644c981a0d8db7b6acd30, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/18dee53d48ee47e8baa49e3d6d90fbee] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=90.7 K 2024-11-11T13:37:38,440 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:38,440 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ee57bb59dc8946ad950aa4378062e17d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b380c4eb34f644c981a0d8db7b6acd30, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/18dee53d48ee47e8baa49e3d6d90fbee] 2024-11-11T13:37:38,442 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting ee57bb59dc8946ad950aa4378062e17d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731332255001 2024-11-11T13:37:38,442 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting c28d12eaab144924b3866d192817ee45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731332255001 2024-11-11T13:37:38,444 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting b380c4eb34f644c981a0d8db7b6acd30, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731332255054 2024-11-11T13:37:38,444 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37eda10f8a6845e8809272e397ce4238, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731332255054 2024-11-11T13:37:38,444 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d93a64744f394b12bdac2995d28746b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731332256203 2024-11-11T13:37:38,444 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 18dee53d48ee47e8baa49e3d6d90fbee, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731332256203 2024-11-11T13:37:38,463 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:38,463 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/c1925122f9a746a79019b03b55731997 is 50, key is test_row_0/C:col10/1731332256213/Put/seqid=0 2024-11-11T13:37:38,464 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:38,476 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411119d023fb3e1a84f528718e1947e5d6155_f177ccda479452e8555a646c67251509 store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:38,478 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411119d023fb3e1a84f528718e1947e5d6155_f177ccda479452e8555a646c67251509, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:38,478 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119d023fb3e1a84f528718e1947e5d6155_f177ccda479452e8555a646c67251509 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:38,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T13:37:38,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:38,479 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T13:37:38,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742388_1564 (size=12104) 2024-11-11T13:37:38,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742389_1565 (size=4469) 2024-11-11T13:37:38,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111167448763db1f4d85bc1cdbb0c744003c_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332257383/Put/seqid=0 2024-11-11T13:37:38,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742390_1566 (size=12154) 2024-11-11T13:37:38,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:38,494 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111167448763db1f4d85bc1cdbb0c744003c_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111167448763db1f4d85bc1cdbb0c744003c_f177ccda479452e8555a646c67251509 2024-11-11T13:37:38,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/85fe4c62d2094ff6a7426230dc3027fc, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:38,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/85fe4c62d2094ff6a7426230dc3027fc is 175, key is test_row_0/A:col10/1731332257383/Put/seqid=0 2024-11-11T13:37:38,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742391_1567 (size=30955) 2024-11-11T13:37:38,498 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/85fe4c62d2094ff6a7426230dc3027fc 2024-11-11T13:37:38,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/e017f4029e2d4451bc10200bde6f8dca is 50, key is test_row_0/B:col10/1731332257383/Put/seqid=0 2024-11-11T13:37:38,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742392_1568 (size=12001) 2024-11-11T13:37:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:38,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:38,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332318538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332318539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332318539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332318546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332318549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332318650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332318659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332318659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332318659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332318861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332318871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332318872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332318872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:38,892 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:38,893 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/400f65a9ee884414a5ac0731a8b52166 is 175, key is test_row_0/A:col10/1731332256213/Put/seqid=0 2024-11-11T13:37:38,914 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/e017f4029e2d4451bc10200bde6f8dca 2024-11-11T13:37:38,924 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/c1925122f9a746a79019b03b55731997 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c1925122f9a746a79019b03b55731997 2024-11-11T13:37:38,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742393_1569 (size=31058) 2024-11-11T13:37:38,950 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/400f65a9ee884414a5ac0731a8b52166 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/400f65a9ee884414a5ac0731a8b52166 2024-11-11T13:37:38,960 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/A of f177ccda479452e8555a646c67251509 into 400f65a9ee884414a5ac0731a8b52166(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:38,960 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:38,960 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/A, priority=13, startTime=1731332258438; duration=0sec 2024-11-11T13:37:38,960 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:38,960 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:A 2024-11-11T13:37:38,960 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:38,961 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:38,961 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/B is initiating minor compaction (all files) 2024-11-11T13:37:38,961 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/B in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:38,961 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/227a7dcaaf704c7881ad19079a461be2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/21535d11bbbb4f5ba09d642fbe8527cb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/c39fdd791fbd450896b03640951888b4] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=35.2 K 2024-11-11T13:37:38,963 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 227a7dcaaf704c7881ad19079a461be2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731332255001 2024-11-11T13:37:38,963 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/C of f177ccda479452e8555a646c67251509 into c1925122f9a746a79019b03b55731997(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:38,964 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:38,964 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/C, priority=13, startTime=1731332258438; duration=0sec 2024-11-11T13:37:38,964 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:38,964 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:C 2024-11-11T13:37:38,964 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 21535d11bbbb4f5ba09d642fbe8527cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731332255054 2024-11-11T13:37:38,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/1c6c2d8da39e40e88dbb8adb0e54a946 is 50, key is test_row_0/C:col10/1731332257383/Put/seqid=0 2024-11-11T13:37:38,964 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c39fdd791fbd450896b03640951888b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731332256203 2024-11-11T13:37:38,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742394_1570 (size=12001) 2024-11-11T13:37:38,992 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#B#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:38,993 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/9c106126e9f64c718d7bf63e6faf2b73 is 50, key is test_row_0/B:col10/1731332256213/Put/seqid=0 2024-11-11T13:37:38,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742395_1571 (size=12104) 2024-11-11T13:37:38,999 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/9c106126e9f64c718d7bf63e6faf2b73 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/9c106126e9f64c718d7bf63e6faf2b73 2024-11-11T13:37:39,003 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/B of f177ccda479452e8555a646c67251509 into 9c106126e9f64c718d7bf63e6faf2b73(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:39,003 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:39,003 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/B, priority=13, startTime=1731332258438; duration=0sec 2024-11-11T13:37:39,003 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:39,003 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:B 2024-11-11T13:37:39,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332319169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:39,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332319179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:39,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332319189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:39,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332319189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T13:37:39,396 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/1c6c2d8da39e40e88dbb8adb0e54a946 2024-11-11T13:37:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/85fe4c62d2094ff6a7426230dc3027fc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/85fe4c62d2094ff6a7426230dc3027fc 2024-11-11T13:37:39,407 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/85fe4c62d2094ff6a7426230dc3027fc, entries=150, sequenceid=76, filesize=30.2 K 2024-11-11T13:37:39,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/e017f4029e2d4451bc10200bde6f8dca as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e017f4029e2d4451bc10200bde6f8dca 2024-11-11T13:37:39,415 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e017f4029e2d4451bc10200bde6f8dca, entries=150, sequenceid=76, filesize=11.7 K 2024-11-11T13:37:39,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/1c6c2d8da39e40e88dbb8adb0e54a946 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/1c6c2d8da39e40e88dbb8adb0e54a946 2024-11-11T13:37:39,420 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/1c6c2d8da39e40e88dbb8adb0e54a946, entries=150, sequenceid=76, filesize=11.7 K 2024-11-11T13:37:39,421 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f177ccda479452e8555a646c67251509 in 942ms, sequenceid=76, compaction requested=false 2024-11-11T13:37:39,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:39,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:39,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-11T13:37:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-11T13:37:39,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-11T13:37:39,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2120 sec 2024-11-11T13:37:39,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.2300 sec 2024-11-11T13:37:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:39,568 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:39,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-11T13:37:39,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b7d6b5d089604bcd8b711ab7bca1bd92_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332258536/Put/seqid=0 2024-11-11T13:37:39,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742396_1572 (size=12154) 2024-11-11T13:37:39,631 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:39,634 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b7d6b5d089604bcd8b711ab7bca1bd92_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b7d6b5d089604bcd8b711ab7bca1bd92_f177ccda479452e8555a646c67251509 2024-11-11T13:37:39,635 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/c01d52731a36433ca7b183afb30aa2e8, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:39,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/c01d52731a36433ca7b183afb30aa2e8 is 175, key is test_row_0/A:col10/1731332258536/Put/seqid=0 2024-11-11T13:37:39,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742397_1573 (size=30955) 2024-11-11T13:37:39,645 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/c01d52731a36433ca7b183afb30aa2e8 2024-11-11T13:37:39,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/e68edb095fbc4122a06a4ed64139a923 is 50, key is test_row_0/B:col10/1731332258536/Put/seqid=0 2024-11-11T13:37:39,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742398_1574 (size=12001) 2024-11-11T13:37:39,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/e68edb095fbc4122a06a4ed64139a923 2024-11-11T13:37:39,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332319679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:39,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/8b010ab83cb44b38af1e86506f8d5004 is 50, key is test_row_0/C:col10/1731332258536/Put/seqid=0 2024-11-11T13:37:39,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332319687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:39,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332319689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:39,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332319700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332319700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:39,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742399_1575 (size=12001) 2024-11-11T13:37:39,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332319789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:40,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332319999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:40,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/8b010ab83cb44b38af1e86506f8d5004 2024-11-11T13:37:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/c01d52731a36433ca7b183afb30aa2e8 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/c01d52731a36433ca7b183afb30aa2e8 2024-11-11T13:37:40,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/c01d52731a36433ca7b183afb30aa2e8, entries=150, sequenceid=94, filesize=30.2 K 2024-11-11T13:37:40,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/e68edb095fbc4122a06a4ed64139a923 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e68edb095fbc4122a06a4ed64139a923 2024-11-11T13:37:40,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e68edb095fbc4122a06a4ed64139a923, entries=150, sequenceid=94, filesize=11.7 K 2024-11-11T13:37:40,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/8b010ab83cb44b38af1e86506f8d5004 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/8b010ab83cb44b38af1e86506f8d5004 2024-11-11T13:37:40,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/8b010ab83cb44b38af1e86506f8d5004, entries=150, sequenceid=94, filesize=11.7 K 2024-11-11T13:37:40,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f177ccda479452e8555a646c67251509 in 593ms, sequenceid=94, compaction requested=true 2024-11-11T13:37:40,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:40,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:40,161 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:40,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:40,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:40,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:40,161 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:40,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:40,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:40,162 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:40,162 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/A is initiating minor compaction (all files) 2024-11-11T13:37:40,162 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/A in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:40,162 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/400f65a9ee884414a5ac0731a8b52166, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/85fe4c62d2094ff6a7426230dc3027fc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/c01d52731a36433ca7b183afb30aa2e8] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=90.8 K 2024-11-11T13:37:40,162 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:40,162 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/400f65a9ee884414a5ac0731a8b52166, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/85fe4c62d2094ff6a7426230dc3027fc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/c01d52731a36433ca7b183afb30aa2e8] 2024-11-11T13:37:40,162 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:40,162 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/B is initiating minor compaction (all files) 2024-11-11T13:37:40,162 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 400f65a9ee884414a5ac0731a8b52166, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731332256203 2024-11-11T13:37:40,162 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/B in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:40,162 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/9c106126e9f64c718d7bf63e6faf2b73, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e017f4029e2d4451bc10200bde6f8dca, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e68edb095fbc4122a06a4ed64139a923] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=35.3 K 2024-11-11T13:37:40,162 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85fe4c62d2094ff6a7426230dc3027fc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731332257382 2024-11-11T13:37:40,163 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c106126e9f64c718d7bf63e6faf2b73, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731332256203 2024-11-11T13:37:40,163 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting c01d52731a36433ca7b183afb30aa2e8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731332258536 2024-11-11T13:37:40,163 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e017f4029e2d4451bc10200bde6f8dca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731332257382 2024-11-11T13:37:40,164 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e68edb095fbc4122a06a4ed64139a923, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731332258536 2024-11-11T13:37:40,172 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:40,177 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#B#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:40,177 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/a1802aa2d79d477cb413d8fa1b5c8db5 is 50, key is test_row_0/B:col10/1731332258536/Put/seqid=0 2024-11-11T13:37:40,179 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111046b30256a734865bf21678207254952_f177ccda479452e8555a646c67251509 store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:40,181 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111046b30256a734865bf21678207254952_f177ccda479452e8555a646c67251509, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:40,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742400_1576 (size=12207) 2024-11-11T13:37:40,181 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111046b30256a734865bf21678207254952_f177ccda479452e8555a646c67251509 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:40,185 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/a1802aa2d79d477cb413d8fa1b5c8db5 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/a1802aa2d79d477cb413d8fa1b5c8db5 2024-11-11T13:37:40,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742401_1577 (size=4469) 2024-11-11T13:37:40,190 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/B of f177ccda479452e8555a646c67251509 into a1802aa2d79d477cb413d8fa1b5c8db5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:40,190 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:40,191 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/B, priority=13, startTime=1731332260161; duration=0sec 2024-11-11T13:37:40,191 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:40,191 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:B 2024-11-11T13:37:40,191 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:40,192 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:40,192 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/C is initiating minor compaction (all files) 2024-11-11T13:37:40,192 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/C in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:40,192 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c1925122f9a746a79019b03b55731997, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/1c6c2d8da39e40e88dbb8adb0e54a946, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/8b010ab83cb44b38af1e86506f8d5004] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=35.3 K 2024-11-11T13:37:40,193 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c1925122f9a746a79019b03b55731997, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731332256203 2024-11-11T13:37:40,193 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c6c2d8da39e40e88dbb8adb0e54a946, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731332257382 2024-11-11T13:37:40,193 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b010ab83cb44b38af1e86506f8d5004, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731332258536 2024-11-11T13:37:40,199 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#C#compaction#497 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:40,199 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/45baaccafa4e447086764bfc6b944b50 is 50, key is test_row_0/C:col10/1731332258536/Put/seqid=0 2024-11-11T13:37:40,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742402_1578 (size=12207) 2024-11-11T13:37:40,208 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/45baaccafa4e447086764bfc6b944b50 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/45baaccafa4e447086764bfc6b944b50 2024-11-11T13:37:40,212 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/C of f177ccda479452e8555a646c67251509 into 45baaccafa4e447086764bfc6b944b50(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:40,213 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:40,213 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/C, priority=13, startTime=1731332260161; duration=0sec 2024-11-11T13:37:40,213 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:40,213 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:C 2024-11-11T13:37:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:40,322 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:40,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-11T13:37:40,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b010ca279fd24aa7a7759e1151e400e5_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332259666/Put/seqid=0 2024-11-11T13:37:40,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742403_1579 (size=14594) 2024-11-11T13:37:40,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332320362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:40,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332320469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:40,594 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#A#compaction#495 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:40,594 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/76b9c572082849f79de34f7e3168b316 is 175, key is test_row_0/A:col10/1731332258536/Put/seqid=0 2024-11-11T13:37:40,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742404_1580 (size=31161) 2024-11-11T13:37:40,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332320683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332320699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332320699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:40,701 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:40,702 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:40,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332320714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:40,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332320714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:40,716 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:40,717 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_2' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:40,735 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:40,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b010ca279fd24aa7a7759e1151e400e5_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b010ca279fd24aa7a7759e1151e400e5_f177ccda479452e8555a646c67251509 2024-11-11T13:37:40,754 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/82e875938e53429db48844120ef30a02, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:40,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/82e875938e53429db48844120ef30a02 is 175, key is test_row_0/A:col10/1731332259666/Put/seqid=0 2024-11-11T13:37:40,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742405_1581 (size=39549) 2024-11-11T13:37:40,788 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/82e875938e53429db48844120ef30a02 2024-11-11T13:37:40,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/adf6693a54234bca9d946b17ef78431e is 50, key is test_row_0/B:col10/1731332259666/Put/seqid=0 2024-11-11T13:37:40,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742406_1582 (size=12001) 2024-11-11T13:37:40,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332320991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:41,009 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/76b9c572082849f79de34f7e3168b316 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/76b9c572082849f79de34f7e3168b316 2024-11-11T13:37:41,013 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/A of f177ccda479452e8555a646c67251509 into 76b9c572082849f79de34f7e3168b316(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:41,013 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:41,013 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/A, priority=13, startTime=1731332260161; duration=0sec 2024-11-11T13:37:41,013 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:41,013 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:A 2024-11-11T13:37:41,161 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T13:37:41,161 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T13:37:41,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/adf6693a54234bca9d946b17ef78431e 2024-11-11T13:37:41,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/ffdcf9c1974a43f199a78ce499a7e2db is 50, key is test_row_0/C:col10/1731332259666/Put/seqid=0 2024-11-11T13:37:41,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742407_1583 (size=12001) 2024-11-11T13:37:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T13:37:41,369 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:41,370 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-11T13:37:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T13:37:41,371 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:41,372 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:41,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T13:37:41,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332321499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:41,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T13:37:41,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:41,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:41,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:41,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:41,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:41,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T13:37:41,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:41,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:41,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:41,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:41,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T13:37:41,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/ffdcf9c1974a43f199a78ce499a7e2db 2024-11-11T13:37:41,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/82e875938e53429db48844120ef30a02 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82e875938e53429db48844120ef30a02 2024-11-11T13:37:41,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82e875938e53429db48844120ef30a02, entries=200, sequenceid=118, filesize=38.6 K 2024-11-11T13:37:41,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/adf6693a54234bca9d946b17ef78431e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/adf6693a54234bca9d946b17ef78431e 2024-11-11T13:37:41,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/adf6693a54234bca9d946b17ef78431e, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T13:37:41,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/ffdcf9c1974a43f199a78ce499a7e2db as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/ffdcf9c1974a43f199a78ce499a7e2db 2024-11-11T13:37:41,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/ffdcf9c1974a43f199a78ce499a7e2db, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T13:37:41,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f177ccda479452e8555a646c67251509 in 1441ms, sequenceid=118, compaction requested=false 2024-11-11T13:37:41,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:41,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T13:37:41,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:41,835 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-11T13:37:41,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111287f79e1e625411092fc987439c17d7c_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332260353/Put/seqid=0 2024-11-11T13:37:41,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742408_1584 (size=12304) 2024-11-11T13:37:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T13:37:42,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:42,258 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111287f79e1e625411092fc987439c17d7c_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111287f79e1e625411092fc987439c17d7c_f177ccda479452e8555a646c67251509 2024-11-11T13:37:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/86827fa657b648b48c6f6189d5181472, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:42,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/86827fa657b648b48c6f6189d5181472 is 175, key is test_row_0/A:col10/1731332260353/Put/seqid=0 2024-11-11T13:37:42,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742409_1585 (size=31105) 2024-11-11T13:37:42,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:42,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T13:37:42,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332322582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:42,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:42,684 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/86827fa657b648b48c6f6189d5181472 2024-11-11T13:37:42,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332322689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/f3110875198f4bab9ba685a164db1b7c is 50, key is test_row_0/B:col10/1731332260353/Put/seqid=0 2024-11-11T13:37:42,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742410_1586 (size=12151) 2024-11-11T13:37:42,717 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/f3110875198f4bab9ba685a164db1b7c 2024-11-11T13:37:42,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332322719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:42,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332322719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:42,720 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,720 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:42,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/c72f34424d7e4eb69a21d9307a6b66f6 is 50, key is test_row_0/C:col10/1731332260353/Put/seqid=0 2024-11-11T13:37:42,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332322729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:42,730 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:42,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332322740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:42,743 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_2' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:42,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742411_1587 (size=12151) 2024-11-11T13:37:42,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332322900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:42,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:43,180 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/c72f34424d7e4eb69a21d9307a6b66f6 2024-11-11T13:37:43,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/86827fa657b648b48c6f6189d5181472 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/86827fa657b648b48c6f6189d5181472 2024-11-11T13:37:43,203 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/86827fa657b648b48c6f6189d5181472, entries=150, sequenceid=133, filesize=30.4 K 2024-11-11T13:37:43,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/f3110875198f4bab9ba685a164db1b7c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f3110875198f4bab9ba685a164db1b7c 2024-11-11T13:37:43,206 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f3110875198f4bab9ba685a164db1b7c, entries=150, sequenceid=133, filesize=11.9 K 2024-11-11T13:37:43,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/c72f34424d7e4eb69a21d9307a6b66f6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c72f34424d7e4eb69a21d9307a6b66f6 2024-11-11T13:37:43,210 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c72f34424d7e4eb69a21d9307a6b66f6, entries=150, sequenceid=133, filesize=11.9 K 2024-11-11T13:37:43,211 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f177ccda479452e8555a646c67251509 in 1376ms, sequenceid=133, compaction requested=true 2024-11-11T13:37:43,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:43,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:43,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-11T13:37:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-11T13:37:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:43,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-11T13:37:43,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-11T13:37:43,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8410 sec 2024-11-11T13:37:43,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b509a8c7f97a422b8ce1a70897acda4e_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332263213/Put/seqid=0 2024-11-11T13:37:43,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.8480 sec 2024-11-11T13:37:43,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742412_1588 (size=14794) 2024-11-11T13:37:43,226 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,229 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b509a8c7f97a422b8ce1a70897acda4e_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b509a8c7f97a422b8ce1a70897acda4e_f177ccda479452e8555a646c67251509 2024-11-11T13:37:43,229 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/6bf1b3441ac2499e968338366bbb2e97, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:43,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/6bf1b3441ac2499e968338366bbb2e97 is 175, key is test_row_0/A:col10/1731332263213/Put/seqid=0 2024-11-11T13:37:43,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742413_1589 (size=39749) 2024-11-11T13:37:43,238 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/6bf1b3441ac2499e968338366bbb2e97 2024-11-11T13:37:43,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/2c84629bb4f848efb2784f741466b1a2 is 50, key is test_row_0/B:col10/1731332263213/Put/seqid=0 2024-11-11T13:37:43,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742414_1590 (size=12151) 2024-11-11T13:37:43,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332323272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:43,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:43,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332323380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T13:37:43,519 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:43,520 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-11T13:37:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T13:37:43,539 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:43,547 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:43,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:43,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332323595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:43,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T13:37:43,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/2c84629bb4f848efb2784f741466b1a2 2024-11-11T13:37:43,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/f93aec0e17504d5d953f63b186d30b7f is 50, key is test_row_0/C:col10/1731332263213/Put/seqid=0 2024-11-11T13:37:43,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742415_1591 (size=12151) 2024-11-11T13:37:43,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/f93aec0e17504d5d953f63b186d30b7f 2024-11-11T13:37:43,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/6bf1b3441ac2499e968338366bbb2e97 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/6bf1b3441ac2499e968338366bbb2e97 2024-11-11T13:37:43,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T13:37:43,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:43,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:43,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:43,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:43,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:43,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/6bf1b3441ac2499e968338366bbb2e97, entries=200, sequenceid=156, filesize=38.8 K 2024-11-11T13:37:43,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/2c84629bb4f848efb2784f741466b1a2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2c84629bb4f848efb2784f741466b1a2 2024-11-11T13:37:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2c84629bb4f848efb2784f741466b1a2, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T13:37:43,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/f93aec0e17504d5d953f63b186d30b7f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/f93aec0e17504d5d953f63b186d30b7f 2024-11-11T13:37:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/f93aec0e17504d5d953f63b186d30b7f, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T13:37:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f177ccda479452e8555a646c67251509 in 501ms, sequenceid=156, compaction requested=true 2024-11-11T13:37:43,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:43,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:43,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:43,715 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:43,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:43,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:43,715 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:43,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:43,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,716 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141564 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:43,716 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/A is initiating minor compaction (all files) 2024-11-11T13:37:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,716 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/A in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,717 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/76b9c572082849f79de34f7e3168b316, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82e875938e53429db48844120ef30a02, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/86827fa657b648b48c6f6189d5181472, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/6bf1b3441ac2499e968338366bbb2e97] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=138.2 K 2024-11-11T13:37:43,717 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:43,717 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/76b9c572082849f79de34f7e3168b316, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82e875938e53429db48844120ef30a02, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/86827fa657b648b48c6f6189d5181472, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/6bf1b3441ac2499e968338366bbb2e97] 2024-11-11T13:37:43,717 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,717 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/B is initiating minor compaction (all files) 2024-11-11T13:37:43,717 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/B in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:43,717 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/a1802aa2d79d477cb413d8fa1b5c8db5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/adf6693a54234bca9d946b17ef78431e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f3110875198f4bab9ba685a164db1b7c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2c84629bb4f848efb2784f741466b1a2] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=47.4 K 2024-11-11T13:37:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,717 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b9c572082849f79de34f7e3168b316, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731332258536 2024-11-11T13:37:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,717 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82e875938e53429db48844120ef30a02, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731332259661 2024-11-11T13:37:43,717 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting a1802aa2d79d477cb413d8fa1b5c8db5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731332258536 2024-11-11T13:37:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,718 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting adf6693a54234bca9d946b17ef78431e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731332259661 2024-11-11T13:37:43,718 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86827fa657b648b48c6f6189d5181472, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731332260353 2024-11-11T13:37:43,718 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f3110875198f4bab9ba685a164db1b7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731332260353 2024-11-11T13:37:43,718 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bf1b3441ac2499e968338366bbb2e97, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332262565 2024-11-11T13:37:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,718 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c84629bb4f848efb2784f741466b1a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332262565 2024-11-11T13:37:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,725 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,726 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#B#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,727 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/ba3812665ce048d1aa2a2a40724360bc is 50, key is test_row_0/B:col10/1731332263213/Put/seqid=0 2024-11-11T13:37:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,727 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110d16c61fe0d6491284fd888516d37110_f177ccda479452e8555a646c67251509 store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,729 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110d16c61fe0d6491284fd888516d37110_f177ccda479452e8555a646c67251509, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,729 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110d16c61fe0d6491284fd888516d37110_f177ccda479452e8555a646c67251509 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742416_1592 (size=12493) 2024-11-11T13:37:43,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742417_1593 (size=4469) 2024-11-11T13:37:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,751 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#A#compaction#508 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,751 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/82151367a2bf4716a13028b02d3908b2 is 175, key is test_row_0/A:col10/1731332263213/Put/seqid=0 2024-11-11T13:37:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,755 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/ba3812665ce048d1aa2a2a40724360bc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/ba3812665ce048d1aa2a2a40724360bc 2024-11-11T13:37:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742418_1594 (size=31447) 2024-11-11T13:37:43,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,778 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f177ccda479452e8555a646c67251509/B of f177ccda479452e8555a646c67251509 into ba3812665ce048d1aa2a2a40724360bc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:43,778 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:43,778 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/B, priority=12, startTime=1731332263715; duration=0sec 2024-11-11T13:37:43,779 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:43,779 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:B 2024-11-11T13:37:43,779 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:43,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,787 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/82151367a2bf4716a13028b02d3908b2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82151367a2bf4716a13028b02d3908b2 2024-11-11T13:37:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,788 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,788 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/C is initiating minor compaction (all files) 2024-11-11T13:37:43,788 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/C in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:43,788 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/45baaccafa4e447086764bfc6b944b50, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/ffdcf9c1974a43f199a78ce499a7e2db, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c72f34424d7e4eb69a21d9307a6b66f6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/f93aec0e17504d5d953f63b186d30b7f] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=47.4 K 2024-11-11T13:37:43,788 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 45baaccafa4e447086764bfc6b944b50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731332258536 2024-11-11T13:37:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,788 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdcf9c1974a43f199a78ce499a7e2db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731332259661 2024-11-11T13:37:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,789 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting c72f34424d7e4eb69a21d9307a6b66f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731332260353 2024-11-11T13:37:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,789 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f93aec0e17504d5d953f63b186d30b7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332262565 2024-11-11T13:37:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,792 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f177ccda479452e8555a646c67251509/A of f177ccda479452e8555a646c67251509 into 82151367a2bf4716a13028b02d3908b2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:43,792 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:43,792 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/A, priority=12, startTime=1731332263715; duration=0sec 2024-11-11T13:37:43,792 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:43,792 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:A 2024-11-11T13:37:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,809 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,810 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/6e3c98503aeb464692a295c553d3a120 is 50, key is test_row_0/C:col10/1731332263213/Put/seqid=0 2024-11-11T13:37:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742419_1595 (size=12493) 2024-11-11T13:37:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,825 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/6e3c98503aeb464692a295c553d3a120 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/6e3c98503aeb464692a295c553d3a120 2024-11-11T13:37:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,830 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f177ccda479452e8555a646c67251509/C of f177ccda479452e8555a646c67251509 into 6e3c98503aeb464692a295c553d3a120(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:43,830 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,830 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/C, priority=12, startTime=1731332263715; duration=0sec 2024-11-11T13:37:43,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,830 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:43,830 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:C 2024-11-11T13:37:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T13:37:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T13:37:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:43,859 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T13:37:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111147d26f2df8044aacb1aca2a14ecef70b_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332263271/Put/seqid=0 2024-11-11T13:37:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742420_1596 (size=9814) 2024-11-11T13:37:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,894 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111147d26f2df8044aacb1aca2a14ecef70b_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111147d26f2df8044aacb1aca2a14ecef70b_f177ccda479452e8555a646c67251509 2024-11-11T13:37:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/8e095b9e20e449df9cd8f25abd40d1df, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/8e095b9e20e449df9cd8f25abd40d1df is 175, key is test_row_0/A:col10/1731332263271/Put/seqid=0 2024-11-11T13:37:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742421_1597 (size=22461) 2024-11-11T13:37:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,914 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/8e095b9e20e449df9cd8f25abd40d1df 2024-11-11T13:37:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/3f5efb51127e40f0b37ae9f1bff9651e is 50, key is test_row_0/B:col10/1731332263271/Put/seqid=0 2024-11-11T13:37:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742422_1598 (size=9757) 2024-11-11T13:37:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:43,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332323993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:44,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332324099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T13:37:44,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332324309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:44,331 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/3f5efb51127e40f0b37ae9f1bff9651e 2024-11-11T13:37:44,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/45ff2bef0b164206827015b1d6d3c5ed is 50, key is test_row_0/C:col10/1731332263271/Put/seqid=0 2024-11-11T13:37:44,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742423_1599 (size=9757) 2024-11-11T13:37:44,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332324619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:44,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T13:37:44,747 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/45ff2bef0b164206827015b1d6d3c5ed 2024-11-11T13:37:44,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/8e095b9e20e449df9cd8f25abd40d1df as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/8e095b9e20e449df9cd8f25abd40d1df 2024-11-11T13:37:44,775 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/8e095b9e20e449df9cd8f25abd40d1df, entries=100, sequenceid=172, filesize=21.9 K 2024-11-11T13:37:44,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/3f5efb51127e40f0b37ae9f1bff9651e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/3f5efb51127e40f0b37ae9f1bff9651e 2024-11-11T13:37:44,786 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/3f5efb51127e40f0b37ae9f1bff9651e, entries=100, sequenceid=172, filesize=9.5 K 2024-11-11T13:37:44,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/45ff2bef0b164206827015b1d6d3c5ed as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/45ff2bef0b164206827015b1d6d3c5ed 2024-11-11T13:37:44,792 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/45ff2bef0b164206827015b1d6d3c5ed, entries=100, sequenceid=172, filesize=9.5 K 2024-11-11T13:37:44,793 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f177ccda479452e8555a646c67251509 in 934ms, sequenceid=172, compaction requested=false 2024-11-11T13:37:44,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:44,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:44,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-11T13:37:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-11T13:37:44,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-11T13:37:44,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2570 sec 2024-11-11T13:37:44,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.3080 sec 2024-11-11T13:37:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:45,132 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:45,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-11T13:37:45,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111161ec6b619898426d95fed473030a7291_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332265131/Put/seqid=0 2024-11-11T13:37:45,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742424_1600 (size=14794) 2024-11-11T13:37:45,142 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:45,145 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111161ec6b619898426d95fed473030a7291_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111161ec6b619898426d95fed473030a7291_f177ccda479452e8555a646c67251509 2024-11-11T13:37:45,145 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/988ea23465f94cf48d73c270ee41d852, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:45,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/988ea23465f94cf48d73c270ee41d852 is 175, key is test_row_0/A:col10/1731332265131/Put/seqid=0 2024-11-11T13:37:45,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742425_1601 (size=39749) 2024-11-11T13:37:45,149 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/988ea23465f94cf48d73c270ee41d852 2024-11-11T13:37:45,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/813a678fd7d84b31b9a32fee48c7cf37 is 50, key is test_row_0/B:col10/1731332265131/Put/seqid=0 2024-11-11T13:37:45,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742426_1602 (size=12151) 2024-11-11T13:37:45,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/813a678fd7d84b31b9a32fee48c7cf37 2024-11-11T13:37:45,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/503d650d7ed3467cbc289c7195f9dd15 is 50, key is test_row_0/C:col10/1731332265131/Put/seqid=0 2024-11-11T13:37:45,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742427_1603 (size=12151) 2024-11-11T13:37:45,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332325167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:45,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332325280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:45,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332325489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:45,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:45,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:45,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:45,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/503d650d7ed3467cbc289c7195f9dd15 2024-11-11T13:37:45,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/988ea23465f94cf48d73c270ee41d852 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/988ea23465f94cf48d73c270ee41d852 2024-11-11T13:37:45,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/988ea23465f94cf48d73c270ee41d852, entries=200, sequenceid=196, filesize=38.8 K 2024-11-11T13:37:45,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/813a678fd7d84b31b9a32fee48c7cf37 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/813a678fd7d84b31b9a32fee48c7cf37 2024-11-11T13:37:45,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/813a678fd7d84b31b9a32fee48c7cf37, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T13:37:45,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/503d650d7ed3467cbc289c7195f9dd15 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/503d650d7ed3467cbc289c7195f9dd15 2024-11-11T13:37:45,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/503d650d7ed3467cbc289c7195f9dd15, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T13:37:45,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f177ccda479452e8555a646c67251509 in 481ms, sequenceid=196, compaction requested=true 2024-11-11T13:37:45,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:45,613 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:45,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:45,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:45,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:45,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:45,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:45,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T13:37:45,614 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:45,623 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:45,623 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:45,624 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/A is initiating minor compaction (all files) 2024-11-11T13:37:45,624 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/B is initiating minor compaction (all files) 2024-11-11T13:37:45,624 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/A in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:45,624 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/B in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:45,624 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82151367a2bf4716a13028b02d3908b2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/8e095b9e20e449df9cd8f25abd40d1df, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/988ea23465f94cf48d73c270ee41d852] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=91.5 K 2024-11-11T13:37:45,624 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/ba3812665ce048d1aa2a2a40724360bc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/3f5efb51127e40f0b37ae9f1bff9651e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/813a678fd7d84b31b9a32fee48c7cf37] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=33.6 K 2024-11-11T13:37:45,624 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:45,624 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82151367a2bf4716a13028b02d3908b2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/8e095b9e20e449df9cd8f25abd40d1df, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/988ea23465f94cf48d73c270ee41d852] 2024-11-11T13:37:45,627 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting ba3812665ce048d1aa2a2a40724360bc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332262565 2024-11-11T13:37:45,631 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82151367a2bf4716a13028b02d3908b2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332262565 2024-11-11T13:37:45,635 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e095b9e20e449df9cd8f25abd40d1df, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731332263250 2024-11-11T13:37:45,635 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f5efb51127e40f0b37ae9f1bff9651e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731332263250 2024-11-11T13:37:45,636 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 813a678fd7d84b31b9a32fee48c7cf37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332263982 2024-11-11T13:37:45,639 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 988ea23465f94cf48d73c270ee41d852, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332263980 2024-11-11T13:37:45,676 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:45,677 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#B#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:45,677 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/5e1715a79d434d39a5bf187f36e23172 is 50, key is test_row_0/B:col10/1731332265131/Put/seqid=0 2024-11-11T13:37:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T13:37:45,682 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:45,683 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-11T13:37:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T13:37:45,700 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411115fcb4057c3954d6b8153b1fea9870204_f177ccda479452e8555a646c67251509 store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:45,702 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411115fcb4057c3954d6b8153b1fea9870204_f177ccda479452e8555a646c67251509, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:45,702 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115fcb4057c3954d6b8153b1fea9870204_f177ccda479452e8555a646c67251509 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:45,703 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:45,706 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:45,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:45,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742428_1604 (size=12595) 2024-11-11T13:37:45,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742429_1605 (size=4469) 2024-11-11T13:37:45,771 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#A#compaction#517 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:45,772 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/77a7b81adc454b0bb143efdbaf4d351c is 175, key is test_row_0/A:col10/1731332265131/Put/seqid=0 2024-11-11T13:37:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T13:37:45,811 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:45,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-11T13:37:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:45,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742430_1606 (size=31549) 2024-11-11T13:37:45,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111195dab5ed3d2b4535b8c4d38d9e8cc380_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332265807/Put/seqid=0 2024-11-11T13:37:45,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T13:37:45,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:45,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:45,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:45,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:45,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:45,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742431_1607 (size=14794) 2024-11-11T13:37:45,887 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/77a7b81adc454b0bb143efdbaf4d351c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/77a7b81adc454b0bb143efdbaf4d351c 2024-11-11T13:37:45,890 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:45,899 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/A of f177ccda479452e8555a646c67251509 into 77a7b81adc454b0bb143efdbaf4d351c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:45,899 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:45,900 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/A, priority=13, startTime=1731332265613; duration=0sec 2024-11-11T13:37:45,900 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:45,900 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:A 2024-11-11T13:37:45,900 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:45,901 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:45,901 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/C is initiating minor compaction (all files) 2024-11-11T13:37:45,901 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/C in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:45,902 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/6e3c98503aeb464692a295c553d3a120, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/45ff2bef0b164206827015b1d6d3c5ed, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/503d650d7ed3467cbc289c7195f9dd15] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=33.6 K 2024-11-11T13:37:45,907 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e3c98503aeb464692a295c553d3a120, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731332262565 2024-11-11T13:37:45,911 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45ff2bef0b164206827015b1d6d3c5ed, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731332263250 2024-11-11T13:37:45,915 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 503d650d7ed3467cbc289c7195f9dd15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332263982 2024-11-11T13:37:45,920 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111195dab5ed3d2b4535b8c4d38d9e8cc380_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111195dab5ed3d2b4535b8c4d38d9e8cc380_f177ccda479452e8555a646c67251509 2024-11-11T13:37:45,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/301942ce2ee7478d92e430d3741803db, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:45,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/301942ce2ee7478d92e430d3741803db is 175, key is test_row_0/A:col10/1731332265807/Put/seqid=0 2024-11-11T13:37:45,945 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#C#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:45,945 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/11d573baf81646d6a665698d13d044b2 is 50, key is test_row_0/C:col10/1731332265131/Put/seqid=0 2024-11-11T13:37:45,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742432_1608 (size=39749) 2024-11-11T13:37:45,947 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/301942ce2ee7478d92e430d3741803db 2024-11-11T13:37:45,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/2ef29b7fd59f4780a3c2f0db3104abb1 is 50, key is test_row_0/B:col10/1731332265807/Put/seqid=0 2024-11-11T13:37:45,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742433_1609 (size=12595) 2024-11-11T13:37:45,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742434_1610 (size=12151) 2024-11-11T13:37:45,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/2ef29b7fd59f4780a3c2f0db3104abb1 2024-11-11T13:37:45,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/9b7c41ac9fae40f1865e4f01bd77fbaa is 50, key is test_row_0/C:col10/1731332265807/Put/seqid=0 2024-11-11T13:37:45,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742435_1611 (size=12151) 2024-11-11T13:37:45,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/9b7c41ac9fae40f1865e4f01bd77fbaa 2024-11-11T13:37:45,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/301942ce2ee7478d92e430d3741803db as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/301942ce2ee7478d92e430d3741803db 2024-11-11T13:37:45,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/301942ce2ee7478d92e430d3741803db, entries=200, sequenceid=209, filesize=38.8 K 2024-11-11T13:37:45,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/2ef29b7fd59f4780a3c2f0db3104abb1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2ef29b7fd59f4780a3c2f0db3104abb1 2024-11-11T13:37:45,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2ef29b7fd59f4780a3c2f0db3104abb1, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T13:37:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/9b7c41ac9fae40f1865e4f01bd77fbaa as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/9b7c41ac9fae40f1865e4f01bd77fbaa 2024-11-11T13:37:45,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/9b7c41ac9fae40f1865e4f01bd77fbaa, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T13:37:45,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=107.34 KB/109920 for f177ccda479452e8555a646c67251509 in 183ms, sequenceid=209, compaction requested=false 2024-11-11T13:37:45,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:45,998 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:45,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T13:37:46,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111655c6513b3c141509aac089d34a237e4_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332265992/Put/seqid=0 2024-11-11T13:37:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T13:37:46,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742436_1612 (size=12304) 2024-11-11T13:37:46,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T13:37:46,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:46,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332326063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:46,132 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/5e1715a79d434d39a5bf187f36e23172 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/5e1715a79d434d39a5bf187f36e23172 2024-11-11T13:37:46,137 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/B of f177ccda479452e8555a646c67251509 into 5e1715a79d434d39a5bf187f36e23172(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:46,137 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:46,137 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/B, priority=13, startTime=1731332265614; duration=0sec 2024-11-11T13:37:46,137 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:46,137 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:B 2024-11-11T13:37:46,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:46,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332326179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:46,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:46,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T13:37:46,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:46,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T13:37:46,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T13:37:46,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:46,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,371 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/11d573baf81646d6a665698d13d044b2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/11d573baf81646d6a665698d13d044b2 2024-11-11T13:37:46,385 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/C of f177ccda479452e8555a646c67251509 into 11d573baf81646d6a665698d13d044b2(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:46,385 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:46,385 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/C, priority=13, startTime=1731332265614; duration=0sec 2024-11-11T13:37:46,385 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:46,385 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:C 2024-11-11T13:37:46,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332326389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:46,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:46,416 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:46,427 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111655c6513b3c141509aac089d34a237e4_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111655c6513b3c141509aac089d34a237e4_f177ccda479452e8555a646c67251509 2024-11-11T13:37:46,427 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b20377159ab84315850c4f053ea14fc4, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:46,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b20377159ab84315850c4f053ea14fc4 is 175, key is test_row_0/A:col10/1731332265992/Put/seqid=0 2024-11-11T13:37:46,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742437_1613 (size=31105) 2024-11-11T13:37:46,447 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b20377159ab84315850c4f053ea14fc4 2024-11-11T13:37:46,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/930bcfb5a46149929942090376ad4832 is 50, key is test_row_0/B:col10/1731332265992/Put/seqid=0 2024-11-11T13:37:46,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742438_1614 (size=12151) 2024-11-11T13:37:46,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/930bcfb5a46149929942090376ad4832 2024-11-11T13:37:46,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T13:37:46,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:46,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/673e21d897e14dbdb7aa1ad81efd78a1 is 50, key is test_row_0/C:col10/1731332265992/Put/seqid=0 2024-11-11T13:37:46,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742439_1615 (size=12151) 2024-11-11T13:37:46,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T13:37:46,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:46,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332326699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:46,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33574 deadline: 1731332326749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:46,755 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_1' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:46,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33622 deadline: 1731332326759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:46,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33620 deadline: 1731332326759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:46,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33560 deadline: 1731332326759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:46,763 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,763 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_0' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,763 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 37d26ecf6460,36877,1731332101576 for 'test_row_2' in f177ccda479452e8555a646c67251509 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T13:37:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:46,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:46,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T13:37:46,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:46,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T13:37:46,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T13:37:46,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:46,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:46,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/673e21d897e14dbdb7aa1ad81efd78a1 2024-11-11T13:37:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b20377159ab84315850c4f053ea14fc4 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b20377159ab84315850c4f053ea14fc4 2024-11-11T13:37:46,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b20377159ab84315850c4f053ea14fc4, entries=150, sequenceid=230, filesize=30.4 K 2024-11-11T13:37:46,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/930bcfb5a46149929942090376ad4832 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/930bcfb5a46149929942090376ad4832 2024-11-11T13:37:46,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/930bcfb5a46149929942090376ad4832, entries=150, sequenceid=230, filesize=11.9 K 2024-11-11T13:37:46,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/673e21d897e14dbdb7aa1ad81efd78a1 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/673e21d897e14dbdb7aa1ad81efd78a1 2024-11-11T13:37:46,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/673e21d897e14dbdb7aa1ad81efd78a1, entries=150, sequenceid=230, filesize=11.9 K 2024-11-11T13:37:47,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for f177ccda479452e8555a646c67251509 in 1002ms, sequenceid=230, compaction requested=true 2024-11-11T13:37:47,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:47,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:47,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:47,000 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:47,000 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:47,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:47,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:47,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:47,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:47,001 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:47,001 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/B is initiating minor compaction (all files) 2024-11-11T13:37:47,001 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/B in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:47,001 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/5e1715a79d434d39a5bf187f36e23172, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2ef29b7fd59f4780a3c2f0db3104abb1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/930bcfb5a46149929942090376ad4832] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=36.0 K 2024-11-11T13:37:47,001 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:47,001 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/A is initiating minor compaction (all files) 2024-11-11T13:37:47,002 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/A in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:47,002 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/77a7b81adc454b0bb143efdbaf4d351c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/301942ce2ee7478d92e430d3741803db, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b20377159ab84315850c4f053ea14fc4] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=100.0 K 2024-11-11T13:37:47,002 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:47,002 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/77a7b81adc454b0bb143efdbaf4d351c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/301942ce2ee7478d92e430d3741803db, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b20377159ab84315850c4f053ea14fc4] 2024-11-11T13:37:47,002 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e1715a79d434d39a5bf187f36e23172, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332263982 2024-11-11T13:37:47,002 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a7b81adc454b0bb143efdbaf4d351c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332263982 2024-11-11T13:37:47,002 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef29b7fd59f4780a3c2f0db3104abb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731332265161 2024-11-11T13:37:47,002 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 301942ce2ee7478d92e430d3741803db, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731332265143 2024-11-11T13:37:47,002 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 930bcfb5a46149929942090376ad4832, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731332265980 2024-11-11T13:37:47,003 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting b20377159ab84315850c4f053ea14fc4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731332265980 2024-11-11T13:37:47,012 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:47,013 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#B#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:47,013 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/1740867675b448739ffdba4ae19774fc is 50, key is test_row_0/B:col10/1731332265992/Put/seqid=0 2024-11-11T13:37:47,027 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111154074a60cb404f71b0a4e51b97da2d3e_f177ccda479452e8555a646c67251509 store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:47,029 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111154074a60cb404f71b0a4e51b97da2d3e_f177ccda479452e8555a646c67251509, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:47,029 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111154074a60cb404f71b0a4e51b97da2d3e_f177ccda479452e8555a646c67251509 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:47,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742440_1616 (size=12697) 2024-11-11T13:37:47,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742441_1617 (size=4469) 2024-11-11T13:37:47,070 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#A#compaction#526 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:47,071 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/e2008215f1904e149f53538283c3a890 is 175, key is test_row_0/A:col10/1731332265992/Put/seqid=0 2024-11-11T13:37:47,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742442_1618 (size=31651) 2024-11-11T13:37:47,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T13:37:47,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:47,116 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T13:37:47,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111593992fe433d4689a4a6a95280d6696e_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332266061/Put/seqid=0 2024-11-11T13:37:47,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742443_1619 (size=12304) 2024-11-11T13:37:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:47,133 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111593992fe433d4689a4a6a95280d6696e_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111593992fe433d4689a4a6a95280d6696e_f177ccda479452e8555a646c67251509 2024-11-11T13:37:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/26860d5a2a06480389d18bfcfe11bdbe, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/26860d5a2a06480389d18bfcfe11bdbe is 175, key is test_row_0/A:col10/1731332266061/Put/seqid=0 2024-11-11T13:37:47,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742444_1620 (size=31105) 2024-11-11T13:37:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:47,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:47,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332327278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:47,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332327389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:47,473 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/1740867675b448739ffdba4ae19774fc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/1740867675b448739ffdba4ae19774fc 2024-11-11T13:37:47,479 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/B of f177ccda479452e8555a646c67251509 into 1740867675b448739ffdba4ae19774fc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:47,479 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:47,480 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/B, priority=13, startTime=1731332267000; duration=0sec 2024-11-11T13:37:47,480 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:47,480 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:B 2024-11-11T13:37:47,480 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:47,481 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:47,481 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/C is initiating minor compaction (all files) 2024-11-11T13:37:47,481 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/C in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:47,481 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/11d573baf81646d6a665698d13d044b2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/9b7c41ac9fae40f1865e4f01bd77fbaa, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/673e21d897e14dbdb7aa1ad81efd78a1] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=36.0 K 2024-11-11T13:37:47,482 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/e2008215f1904e149f53538283c3a890 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/e2008215f1904e149f53538283c3a890 2024-11-11T13:37:47,482 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 11d573baf81646d6a665698d13d044b2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731332263982 2024-11-11T13:37:47,483 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b7c41ac9fae40f1865e4f01bd77fbaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731332265161 2024-11-11T13:37:47,483 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 673e21d897e14dbdb7aa1ad81efd78a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731332265980 2024-11-11T13:37:47,492 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/A of f177ccda479452e8555a646c67251509 into e2008215f1904e149f53538283c3a890(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:47,492 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:47,492 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/A, priority=13, startTime=1731332267000; duration=0sec 2024-11-11T13:37:47,492 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:47,492 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:A 2024-11-11T13:37:47,508 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#C#compaction#528 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:47,508 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/84b7bedebc67455aaad9e0ce70e0d962 is 50, key is test_row_0/C:col10/1731332265992/Put/seqid=0 2024-11-11T13:37:47,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742445_1621 (size=12697) 2024-11-11T13:37:47,538 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/26860d5a2a06480389d18bfcfe11bdbe 2024-11-11T13:37:47,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/efb21cef4334472281875400c5397a58 is 50, key is test_row_0/B:col10/1731332266061/Put/seqid=0 2024-11-11T13:37:47,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742446_1622 (size=12151) 2024-11-11T13:37:47,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332327603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T13:37:47,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332327920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:47,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:47,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:47,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:47,929 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/84b7bedebc67455aaad9e0ce70e0d962 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/84b7bedebc67455aaad9e0ce70e0d962 2024-11-11T13:37:47,933 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/C of f177ccda479452e8555a646c67251509 into 84b7bedebc67455aaad9e0ce70e0d962(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:47,933 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:47,933 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/C, priority=13, startTime=1731332267000; duration=0sec 2024-11-11T13:37:47,933 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:47,933 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:C 2024-11-11T13:37:47,981 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/efb21cef4334472281875400c5397a58 2024-11-11T13:37:47,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/65c22dfb39784f1b9961dde666eacb72 is 50, key is test_row_0/C:col10/1731332266061/Put/seqid=0 2024-11-11T13:37:48,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742447_1623 (size=12151) 2024-11-11T13:37:48,026 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/65c22dfb39784f1b9961dde666eacb72 2024-11-11T13:37:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/26860d5a2a06480389d18bfcfe11bdbe as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/26860d5a2a06480389d18bfcfe11bdbe 2024-11-11T13:37:48,064 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/26860d5a2a06480389d18bfcfe11bdbe, entries=150, sequenceid=248, filesize=30.4 K 2024-11-11T13:37:48,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/efb21cef4334472281875400c5397a58 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/efb21cef4334472281875400c5397a58 2024-11-11T13:37:48,081 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/efb21cef4334472281875400c5397a58, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T13:37:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/65c22dfb39784f1b9961dde666eacb72 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/65c22dfb39784f1b9961dde666eacb72 2024-11-11T13:37:48,104 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/65c22dfb39784f1b9961dde666eacb72, entries=150, sequenceid=248, filesize=11.9 K 2024-11-11T13:37:48,115 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for f177ccda479452e8555a646c67251509 in 999ms, sequenceid=248, compaction requested=false 2024-11-11T13:37:48,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:48,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:48,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-11T13:37:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-11T13:37:48,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-11T13:37:48,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4110 sec 2024-11-11T13:37:48,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.4380 sec 2024-11-11T13:37:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:48,432 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:48,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-11T13:37:48,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111963800a9d1644498bb42a10d48acd9f0_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332267271/Put/seqid=0 2024-11-11T13:37:48,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742448_1624 (size=12454) 2024-11-11T13:37:48,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332328478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:48,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332328589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:48,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332328799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:48,849 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:48,852 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111963800a9d1644498bb42a10d48acd9f0_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111963800a9d1644498bb42a10d48acd9f0_f177ccda479452e8555a646c67251509 2024-11-11T13:37:48,853 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b523a06c5cd8417db753f5e026ce4e1c, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:48,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b523a06c5cd8417db753f5e026ce4e1c is 175, key is test_row_0/A:col10/1731332267271/Put/seqid=0 2024-11-11T13:37:48,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742449_1625 (size=31255) 2024-11-11T13:37:48,860 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b523a06c5cd8417db753f5e026ce4e1c 2024-11-11T13:37:48,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/f8673c71200b49dea53ef29a19c78da2 is 50, key is test_row_0/B:col10/1731332267271/Put/seqid=0 2024-11-11T13:37:48,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742450_1626 (size=12301) 2024-11-11T13:37:48,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/f8673c71200b49dea53ef29a19c78da2 2024-11-11T13:37:48,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/4f9a58b69112440fa03623d6840d3aff is 50, key is test_row_0/C:col10/1731332267271/Put/seqid=0 2024-11-11T13:37:48,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742451_1627 (size=12301) 2024-11-11T13:37:49,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332329111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:49,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/4f9a58b69112440fa03623d6840d3aff 2024-11-11T13:37:49,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/b523a06c5cd8417db753f5e026ce4e1c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b523a06c5cd8417db753f5e026ce4e1c 2024-11-11T13:37:49,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b523a06c5cd8417db753f5e026ce4e1c, entries=150, sequenceid=272, filesize=30.5 K 2024-11-11T13:37:49,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/f8673c71200b49dea53ef29a19c78da2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f8673c71200b49dea53ef29a19c78da2 2024-11-11T13:37:49,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f8673c71200b49dea53ef29a19c78da2, entries=150, sequenceid=272, filesize=12.0 K 2024-11-11T13:37:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/4f9a58b69112440fa03623d6840d3aff as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/4f9a58b69112440fa03623d6840d3aff 2024-11-11T13:37:49,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/4f9a58b69112440fa03623d6840d3aff, entries=150, sequenceid=272, filesize=12.0 K 2024-11-11T13:37:49,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for f177ccda479452e8555a646c67251509 in 871ms, sequenceid=272, compaction requested=true 2024-11-11T13:37:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:49,303 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:49,303 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:49,304 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:49,304 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/A is initiating minor compaction (all files) 2024-11-11T13:37:49,304 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/A in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:49,304 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/e2008215f1904e149f53538283c3a890, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/26860d5a2a06480389d18bfcfe11bdbe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b523a06c5cd8417db753f5e026ce4e1c] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=91.8 K 2024-11-11T13:37:49,304 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:49,304 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/e2008215f1904e149f53538283c3a890, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/26860d5a2a06480389d18bfcfe11bdbe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b523a06c5cd8417db753f5e026ce4e1c] 2024-11-11T13:37:49,304 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:49,304 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/B is initiating minor compaction (all files) 2024-11-11T13:37:49,304 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e2008215f1904e149f53538283c3a890, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731332265980 2024-11-11T13:37:49,304 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/B in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:49,304 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/1740867675b448739ffdba4ae19774fc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/efb21cef4334472281875400c5397a58, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f8673c71200b49dea53ef29a19c78da2] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=36.3 K 2024-11-11T13:37:49,304 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 26860d5a2a06480389d18bfcfe11bdbe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731332266048 2024-11-11T13:37:49,304 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1740867675b448739ffdba4ae19774fc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731332265980 2024-11-11T13:37:49,305 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting b523a06c5cd8417db753f5e026ce4e1c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731332267271 2024-11-11T13:37:49,305 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb21cef4334472281875400c5397a58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731332266048 2024-11-11T13:37:49,305 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8673c71200b49dea53ef29a19c78da2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731332267271 2024-11-11T13:37:49,315 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:49,316 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111dd4162c4535144a9b3442111ba94f63e_f177ccda479452e8555a646c67251509 store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:49,317 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#B#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:49,317 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/2d3a50a82d93463b909c9391193136a6 is 50, key is test_row_0/B:col10/1731332267271/Put/seqid=0 2024-11-11T13:37:49,318 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111dd4162c4535144a9b3442111ba94f63e_f177ccda479452e8555a646c67251509, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:49,318 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dd4162c4535144a9b3442111ba94f63e_f177ccda479452e8555a646c67251509 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:49,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742452_1628 (size=12949) 2024-11-11T13:37:49,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742453_1629 (size=4469) 2024-11-11T13:37:49,325 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#A#compaction#534 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:49,326 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/ff9e48997a94448dbb65074b5fd8c1da is 175, key is test_row_0/A:col10/1731332267271/Put/seqid=0 2024-11-11T13:37:49,329 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/2d3a50a82d93463b909c9391193136a6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2d3a50a82d93463b909c9391193136a6 2024-11-11T13:37:49,332 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/B of f177ccda479452e8555a646c67251509 into 2d3a50a82d93463b909c9391193136a6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:49,332 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:49,332 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/B, priority=13, startTime=1731332269303; duration=0sec 2024-11-11T13:37:49,333 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:49,333 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:B 2024-11-11T13:37:49,333 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:49,334 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:49,334 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/C is initiating minor compaction (all files) 2024-11-11T13:37:49,334 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/C in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:49,334 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/84b7bedebc67455aaad9e0ce70e0d962, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/65c22dfb39784f1b9961dde666eacb72, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/4f9a58b69112440fa03623d6840d3aff] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=36.3 K 2024-11-11T13:37:49,334 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84b7bedebc67455aaad9e0ce70e0d962, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731332265980 2024-11-11T13:37:49,334 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65c22dfb39784f1b9961dde666eacb72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731332266048 2024-11-11T13:37:49,335 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f9a58b69112440fa03623d6840d3aff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731332267271 2024-11-11T13:37:49,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742454_1630 (size=31903) 2024-11-11T13:37:49,341 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#C#compaction#536 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:49,341 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/d92b1be078e9499298f8ed7e635b03fb is 50, key is test_row_0/C:col10/1731332267271/Put/seqid=0 2024-11-11T13:37:49,341 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/ff9e48997a94448dbb65074b5fd8c1da as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ff9e48997a94448dbb65074b5fd8c1da 2024-11-11T13:37:49,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742455_1631 (size=12949) 2024-11-11T13:37:49,345 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/A of f177ccda479452e8555a646c67251509 into ff9e48997a94448dbb65074b5fd8c1da(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:49,345 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:49,345 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/A, priority=13, startTime=1731332269303; duration=0sec 2024-11-11T13:37:49,345 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:49,345 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:A 2024-11-11T13:37:49,348 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/d92b1be078e9499298f8ed7e635b03fb as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/d92b1be078e9499298f8ed7e635b03fb 2024-11-11T13:37:49,352 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/C of f177ccda479452e8555a646c67251509 into d92b1be078e9499298f8ed7e635b03fb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:49,352 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:49,352 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/C, priority=13, startTime=1731332269303; duration=0sec 2024-11-11T13:37:49,352 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:49,352 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:C 2024-11-11T13:37:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:49,634 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:49,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-11T13:37:49,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114b019f1ab51d402eaccf491173a287b8_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332269630/Put/seqid=0 2024-11-11T13:37:49,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742456_1632 (size=14994) 2024-11-11T13:37:49,660 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:49,666 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114b019f1ab51d402eaccf491173a287b8_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b019f1ab51d402eaccf491173a287b8_f177ccda479452e8555a646c67251509 2024-11-11T13:37:49,668 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/640387c3b18d4f3d8844b99170b11165, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:49,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/640387c3b18d4f3d8844b99170b11165 is 175, key is test_row_0/A:col10/1731332269630/Put/seqid=0 2024-11-11T13:37:49,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742457_1633 (size=39949) 2024-11-11T13:37:49,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332329702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:49,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:49,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332329809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T13:37:49,880 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:49,880 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-11T13:37:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T13:37:49,887 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:49,888 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:49,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T13:37:50,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332330022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:50,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T13:37:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,082 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/640387c3b18d4f3d8844b99170b11165 2024-11-11T13:37:50,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/d24176c049914cc7a20e7e531cf709de is 50, key is test_row_0/B:col10/1731332269630/Put/seqid=0 2024-11-11T13:37:50,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742458_1634 (size=12301) 2024-11-11T13:37:50,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T13:37:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T13:37:50,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:50,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332330329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:50,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T13:37:50,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:50,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T13:37:50,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:50,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T13:37:50,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/d24176c049914cc7a20e7e531cf709de 2024-11-11T13:37:50,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/bb140286afe84fb0b2f8e39b8ae5ae0f is 50, key is test_row_0/C:col10/1731332269630/Put/seqid=0 2024-11-11T13:37:50,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742459_1635 (size=12301) 2024-11-11T13:37:50,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T13:37:50,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:50,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T13:37:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332330839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:50,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T13:37:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:50,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:51,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/bb140286afe84fb0b2f8e39b8ae5ae0f 2024-11-11T13:37:51,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/640387c3b18d4f3d8844b99170b11165 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/640387c3b18d4f3d8844b99170b11165 2024-11-11T13:37:51,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/640387c3b18d4f3d8844b99170b11165, entries=200, sequenceid=291, filesize=39.0 K 2024-11-11T13:37:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/d24176c049914cc7a20e7e531cf709de as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d24176c049914cc7a20e7e531cf709de 2024-11-11T13:37:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T13:37:51,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d24176c049914cc7a20e7e531cf709de, entries=150, sequenceid=291, filesize=12.0 K 2024-11-11T13:37:51,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/bb140286afe84fb0b2f8e39b8ae5ae0f as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/bb140286afe84fb0b2f8e39b8ae5ae0f 2024-11-11T13:37:51,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/bb140286afe84fb0b2f8e39b8ae5ae0f, entries=150, sequenceid=291, filesize=12.0 K 2024-11-11T13:37:51,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for f177ccda479452e8555a646c67251509 in 1401ms, sequenceid=291, compaction requested=false 2024-11-11T13:37:51,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:51,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T13:37:51,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:51,123 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-11T13:37:51,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115e59e7d66e57483caa7f6db7e2b53c99_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332269694/Put/seqid=0 2024-11-11T13:37:51,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742460_1636 (size=12454) 2024-11-11T13:37:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,612 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115e59e7d66e57483caa7f6db7e2b53c99_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115e59e7d66e57483caa7f6db7e2b53c99_f177ccda479452e8555a646c67251509 2024-11-11T13:37:51,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/d4af933c4dca48d798872ebc67d7b2d6, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/d4af933c4dca48d798872ebc67d7b2d6 is 175, key is test_row_0/A:col10/1731332269694/Put/seqid=0 2024-11-11T13:37:51,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742461_1637 (size=31255) 2024-11-11T13:37:51,623 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/d4af933c4dca48d798872ebc67d7b2d6 2024-11-11T13:37:51,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/7e49631f23ed48be88be3199fb866e26 is 50, key is test_row_0/B:col10/1731332269694/Put/seqid=0 2024-11-11T13:37:51,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742462_1638 (size=12301) 2024-11-11T13:37:51,639 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/7e49631f23ed48be88be3199fb866e26 2024-11-11T13:37:51,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/020cc4fe6da2486d81364e9163582a8b is 50, key is test_row_0/C:col10/1731332269694/Put/seqid=0 2024-11-11T13:37:51,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742463_1639 (size=12301) 2024-11-11T13:37:51,664 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/020cc4fe6da2486d81364e9163582a8b 2024-11-11T13:37:51,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/d4af933c4dca48d798872ebc67d7b2d6 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/d4af933c4dca48d798872ebc67d7b2d6 2024-11-11T13:37:51,670 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/d4af933c4dca48d798872ebc67d7b2d6, entries=150, sequenceid=311, filesize=30.5 K 2024-11-11T13:37:51,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/7e49631f23ed48be88be3199fb866e26 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/7e49631f23ed48be88be3199fb866e26 2024-11-11T13:37:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,674 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/7e49631f23ed48be88be3199fb866e26, entries=150, sequenceid=311, filesize=12.0 K 2024-11-11T13:37:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/020cc4fe6da2486d81364e9163582a8b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/020cc4fe6da2486d81364e9163582a8b 2024-11-11T13:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,678 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/020cc4fe6da2486d81364e9163582a8b, entries=150, sequenceid=311, filesize=12.0 K 2024-11-11T13:37:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,679 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for f177ccda479452e8555a646c67251509 in 555ms, sequenceid=311, compaction requested=true 2024-11-11T13:37:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:51,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-11T13:37:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-11T13:37:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-11T13:37:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7920 sec 2024-11-11T13:37:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8030 sec 2024-11-11T13:37:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,958 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:51,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-11T13:37:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111dbb46564cb144e6a27d5a06712ab149_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332271948/Put/seqid=0 2024-11-11T13:37:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:52,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742464_1640 (size=14994) 2024-11-11T13:37:52,008 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:52,011 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111dbb46564cb144e6a27d5a06712ab149_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111dbb46564cb144e6a27d5a06712ab149_f177ccda479452e8555a646c67251509 2024-11-11T13:37:52,012 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/898931e4f1fc48b8b8271964cd2615f2, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:52,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/898931e4f1fc48b8b8271964cd2615f2 is 175, key is test_row_0/A:col10/1731332271948/Put/seqid=0 2024-11-11T13:37:52,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742465_1641 (size=39949) 2024-11-11T13:37:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T13:37:52,040 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:52,041 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T13:37:52,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-11T13:37:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T13:37:52,042 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T13:37:52,043 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T13:37:52,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T13:37:52,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332332110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T13:37:52,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T13:37:52,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:52,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332332223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:52,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T13:37:52,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:52,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T13:37:52,429 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/898931e4f1fc48b8b8271964cd2615f2 2024-11-11T13:37:52,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/ff6e71266e8f41c09207f5d815e08d80 is 50, key is test_row_0/B:col10/1731332271948/Put/seqid=0 2024-11-11T13:37:52,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742466_1642 (size=12301) 2024-11-11T13:37:52,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/ff6e71266e8f41c09207f5d815e08d80 2024-11-11T13:37:52,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332332439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:52,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:52,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:52,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/0afcead517814f39801e03641c762d36 is 50, key is test_row_0/C:col10/1731332271948/Put/seqid=0 2024-11-11T13:37:52,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742467_1643 (size=12301) 2024-11-11T13:37:52,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T13:37:52,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:52,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T13:37:52,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:52,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T13:37:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332332749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:52,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T13:37:52,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:52,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:37:52,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/0afcead517814f39801e03641c762d36 2024-11-11T13:37:52,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/898931e4f1fc48b8b8271964cd2615f2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/898931e4f1fc48b8b8271964cd2615f2 2024-11-11T13:37:52,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/898931e4f1fc48b8b8271964cd2615f2, entries=200, sequenceid=322, filesize=39.0 K 2024-11-11T13:37:52,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/ff6e71266e8f41c09207f5d815e08d80 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/ff6e71266e8f41c09207f5d815e08d80 2024-11-11T13:37:52,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/ff6e71266e8f41c09207f5d815e08d80, entries=150, sequenceid=322, filesize=12.0 K 2024-11-11T13:37:52,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/0afcead517814f39801e03641c762d36 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/0afcead517814f39801e03641c762d36 2024-11-11T13:37:52,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/0afcead517814f39801e03641c762d36, entries=150, sequenceid=322, filesize=12.0 K 2024-11-11T13:37:52,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f177ccda479452e8555a646c67251509 in 931ms, sequenceid=322, compaction requested=true 2024-11-11T13:37:52,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:52,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:52,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:52,889 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:52,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:52,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:52,889 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:52,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:52,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:52,891 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:52,891 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/B is initiating minor compaction (all files) 2024-11-11T13:37:52,891 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/B in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,891 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2d3a50a82d93463b909c9391193136a6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d24176c049914cc7a20e7e531cf709de, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/7e49631f23ed48be88be3199fb866e26, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/ff6e71266e8f41c09207f5d815e08d80] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=48.7 K 2024-11-11T13:37:52,891 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:52,891 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/A is initiating minor compaction (all files) 2024-11-11T13:37:52,891 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/A in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,891 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ff9e48997a94448dbb65074b5fd8c1da, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/640387c3b18d4f3d8844b99170b11165, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/d4af933c4dca48d798872ebc67d7b2d6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/898931e4f1fc48b8b8271964cd2615f2] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=139.7 K 2024-11-11T13:37:52,891 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,891 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ff9e48997a94448dbb65074b5fd8c1da, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/640387c3b18d4f3d8844b99170b11165, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/d4af933c4dca48d798872ebc67d7b2d6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/898931e4f1fc48b8b8271964cd2615f2] 2024-11-11T13:37:52,892 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff9e48997a94448dbb65074b5fd8c1da, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731332267271 2024-11-11T13:37:52,892 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d3a50a82d93463b909c9391193136a6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731332267271 2024-11-11T13:37:52,892 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 640387c3b18d4f3d8844b99170b11165, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731332268460 2024-11-11T13:37:52,892 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting d24176c049914cc7a20e7e531cf709de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731332268469 2024-11-11T13:37:52,892 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e49631f23ed48be88be3199fb866e26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731332269694 2024-11-11T13:37:52,892 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4af933c4dca48d798872ebc67d7b2d6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731332269694 2024-11-11T13:37:52,892 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting ff6e71266e8f41c09207f5d815e08d80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731332271931 2024-11-11T13:37:52,892 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 898931e4f1fc48b8b8271964cd2615f2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731332271931 2024-11-11T13:37:52,900 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:52,902 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#B#compaction#547 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:52,903 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/f21f7266a43141668d97fdad735eff02 is 50, key is test_row_0/B:col10/1731332271948/Put/seqid=0 2024-11-11T13:37:52,908 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111e6d6b1a2847a4baba7ddb45355ba45ed_f177ccda479452e8555a646c67251509 store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:52,910 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111e6d6b1a2847a4baba7ddb45355ba45ed_f177ccda479452e8555a646c67251509, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:52,910 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e6d6b1a2847a4baba7ddb45355ba45ed_f177ccda479452e8555a646c67251509 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:52,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742468_1644 (size=13085) 2024-11-11T13:37:52,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742469_1645 (size=4469) 2024-11-11T13:37:52,918 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#A#compaction#546 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:52,918 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/f921075231ed46e9bc0e168b1ebe29bc is 175, key is test_row_0/A:col10/1731332271948/Put/seqid=0 2024-11-11T13:37:52,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742470_1646 (size=32039) 2024-11-11T13:37:52,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T13:37:52,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:52,971 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:37:52,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111132a4720dcd18419a97cd0967cfd709c2_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332272096/Put/seqid=0 2024-11-11T13:37:52,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742471_1647 (size=12454) 2024-11-11T13:37:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T13:37:53,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. as already flushing 2024-11-11T13:37:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:53,321 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/f21f7266a43141668d97fdad735eff02 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f21f7266a43141668d97fdad735eff02 2024-11-11T13:37:53,327 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f177ccda479452e8555a646c67251509/B of f177ccda479452e8555a646c67251509 into f21f7266a43141668d97fdad735eff02(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:53,327 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:53,327 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/B, priority=12, startTime=1731332272889; duration=0sec 2024-11-11T13:37:53,327 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:53,327 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:B 2024-11-11T13:37:53,327 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T13:37:53,328 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T13:37:53,328 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/C is initiating minor compaction (all files) 2024-11-11T13:37:53,328 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/C in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:53,328 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/d92b1be078e9499298f8ed7e635b03fb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/bb140286afe84fb0b2f8e39b8ae5ae0f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/020cc4fe6da2486d81364e9163582a8b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/0afcead517814f39801e03641c762d36] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=48.7 K 2024-11-11T13:37:53,328 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting d92b1be078e9499298f8ed7e635b03fb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731332267271 2024-11-11T13:37:53,329 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting bb140286afe84fb0b2f8e39b8ae5ae0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731332268469 2024-11-11T13:37:53,329 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 020cc4fe6da2486d81364e9163582a8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731332269694 2024-11-11T13:37:53,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332333325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:53,330 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 0afcead517814f39801e03641c762d36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731332271931 2024-11-11T13:37:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:53,331 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/f921075231ed46e9bc0e168b1ebe29bc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/f921075231ed46e9bc0e168b1ebe29bc 2024-11-11T13:37:53,335 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f177ccda479452e8555a646c67251509/A of f177ccda479452e8555a646c67251509 into f921075231ed46e9bc0e168b1ebe29bc(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:53,335 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:53,335 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/A, priority=12, startTime=1731332272889; duration=0sec 2024-11-11T13:37:53,335 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:53,335 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:A 2024-11-11T13:37:53,338 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#C#compaction#549 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:53,339 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/91aa629eb9134a4a8b0b772008e7249a is 50, key is test_row_0/C:col10/1731332271948/Put/seqid=0 2024-11-11T13:37:53,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742472_1648 (size=13085) 2024-11-11T13:37:53,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:53,385 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111132a4720dcd18419a97cd0967cfd709c2_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111132a4720dcd18419a97cd0967cfd709c2_f177ccda479452e8555a646c67251509 2024-11-11T13:37:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/084ced468747491b85078e09a67a81dc, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/084ced468747491b85078e09a67a81dc is 175, key is test_row_0/A:col10/1731332272096/Put/seqid=0 2024-11-11T13:37:53,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742473_1649 (size=31255) 2024-11-11T13:37:53,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332333439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:53,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:53,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:53,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332333649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:53,753 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/91aa629eb9134a4a8b0b772008e7249a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/91aa629eb9134a4a8b0b772008e7249a 2024-11-11T13:37:53,757 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f177ccda479452e8555a646c67251509/C of f177ccda479452e8555a646c67251509 into 91aa629eb9134a4a8b0b772008e7249a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:53,757 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:53,757 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/C, priority=12, startTime=1731332272889; duration=0sec 2024-11-11T13:37:53,757 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:53,757 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:C 2024-11-11T13:37:53,792 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/084ced468747491b85078e09a67a81dc 2024-11-11T13:37:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/d488a68bb6f243d897ea7cad46e7373b is 50, key is test_row_0/B:col10/1731332272096/Put/seqid=0 2024-11-11T13:37:53,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742474_1650 (size=12301) 2024-11-11T13:37:53,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332333959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T13:37:54,261 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/d488a68bb6f243d897ea7cad46e7373b 2024-11-11T13:37:54,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/e26d27762b6747b9b0103989c27f2d75 is 50, key is test_row_0/C:col10/1731332272096/Put/seqid=0 2024-11-11T13:37:54,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742475_1651 (size=12301) 2024-11-11T13:37:54,309 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/e26d27762b6747b9b0103989c27f2d75 2024-11-11T13:37:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/084ced468747491b85078e09a67a81dc as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/084ced468747491b85078e09a67a81dc 2024-11-11T13:37:54,319 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/084ced468747491b85078e09a67a81dc, entries=150, sequenceid=347, filesize=30.5 K 2024-11-11T13:37:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/d488a68bb6f243d897ea7cad46e7373b as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d488a68bb6f243d897ea7cad46e7373b 2024-11-11T13:37:54,325 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d488a68bb6f243d897ea7cad46e7373b, entries=150, sequenceid=347, filesize=12.0 K 2024-11-11T13:37:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/e26d27762b6747b9b0103989c27f2d75 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/e26d27762b6747b9b0103989c27f2d75 2024-11-11T13:37:54,331 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/e26d27762b6747b9b0103989c27f2d75, entries=150, sequenceid=347, filesize=12.0 K 2024-11-11T13:37:54,332 INFO [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f177ccda479452e8555a646c67251509 in 1361ms, sequenceid=347, compaction requested=false 2024-11-11T13:37:54,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:54,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:54,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-11T13:37:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-11T13:37:54,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-11T13:37:54,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2930 sec 2024-11-11T13:37:54,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.3030 sec 2024-11-11T13:37:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:54,471 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:54,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-11T13:37:54,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111566beefa6aa84370bb7c9c792e233dcb_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332273316/Put/seqid=0 2024-11-11T13:37:54,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742476_1652 (size=14994) 2024-11-11T13:37:54,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332334544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:54,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:54,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:54,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:54,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332334649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:54,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332334867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:54,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:54,884 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:54,890 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111566beefa6aa84370bb7c9c792e233dcb_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111566beefa6aa84370bb7c9c792e233dcb_f177ccda479452e8555a646c67251509 2024-11-11T13:37:54,892 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/257235c3a86f46e681cbff3ba9e8f8c2, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:54,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/257235c3a86f46e681cbff3ba9e8f8c2 is 175, key is test_row_0/A:col10/1731332273316/Put/seqid=0 2024-11-11T13:37:54,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742477_1653 (size=39949) 2024-11-11T13:37:54,905 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/257235c3a86f46e681cbff3ba9e8f8c2 2024-11-11T13:37:54,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/442d4c0c831b4586be093d580c08b5bf is 50, key is test_row_0/B:col10/1731332273316/Put/seqid=0 2024-11-11T13:37:54,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742478_1654 (size=12301) 2024-11-11T13:37:54,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/442d4c0c831b4586be093d580c08b5bf 2024-11-11T13:37:54,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/320ea84ebd9140ee977b6733362005f3 is 50, key is test_row_0/C:col10/1731332273316/Put/seqid=0 2024-11-11T13:37:54,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742479_1655 (size=12301) 2024-11-11T13:37:55,049 INFO [Thread-2471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2471. 2024-11-11T13:37:55,049 DEBUG [Thread-2471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:55,049 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,049 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:55,051 INFO [Thread-2467 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2467. 2024-11-11T13:37:55,051 DEBUG [Thread-2467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:55,051 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,051 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:55,052 INFO [Thread-2463 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2463. 2024-11-11T13:37:55,052 DEBUG [Thread-2463 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:55,052 DEBUG [Thread-2463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,052 DEBUG [Thread-2463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,053 INFO [Thread-2469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2469. 2024-11-11T13:37:55,053 DEBUG [Thread-2469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:55,053 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,053 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:55,053 INFO [Thread-2465 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2465. 2024-11-11T13:37:55,053 DEBUG [Thread-2465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:55,053 DEBUG [Thread-2465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,054 DEBUG [Thread-2465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:55,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:55,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T13:37:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33606 deadline: 1731332335179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 , the old value is region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T13:37:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., hostname=37d26ecf6460,36877,1731332101576, seqNum=5 because the exception is null or not the one we care about 2024-11-11T13:37:55,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/320ea84ebd9140ee977b6733362005f3 2024-11-11T13:37:55,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/257235c3a86f46e681cbff3ba9e8f8c2 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/257235c3a86f46e681cbff3ba9e8f8c2 2024-11-11T13:37:55,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/257235c3a86f46e681cbff3ba9e8f8c2, entries=200, sequenceid=362, filesize=39.0 K 2024-11-11T13:37:55,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/442d4c0c831b4586be093d580c08b5bf as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/442d4c0c831b4586be093d580c08b5bf 2024-11-11T13:37:55,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/442d4c0c831b4586be093d580c08b5bf, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T13:37:55,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/320ea84ebd9140ee977b6733362005f3 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/320ea84ebd9140ee977b6733362005f3 2024-11-11T13:37:55,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/320ea84ebd9140ee977b6733362005f3, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T13:37:55,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f177ccda479452e8555a646c67251509 in 909ms, sequenceid=362, compaction requested=true 2024-11-11T13:37:55,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:55,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T13:37:55,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:55,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T13:37:55,380 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:55,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:55,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f177ccda479452e8555a646c67251509:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T13:37:55,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:55,380 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:55,381 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:55,381 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:55,382 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/B is initiating minor compaction (all files) 2024-11-11T13:37:55,382 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/A is initiating minor compaction (all files) 2024-11-11T13:37:55,382 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/B in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:55,382 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/A in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:55,382 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f21f7266a43141668d97fdad735eff02, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d488a68bb6f243d897ea7cad46e7373b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/442d4c0c831b4586be093d580c08b5bf] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=36.8 K 2024-11-11T13:37:55,382 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/f921075231ed46e9bc0e168b1ebe29bc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/084ced468747491b85078e09a67a81dc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/257235c3a86f46e681cbff3ba9e8f8c2] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=100.8 K 2024-11-11T13:37:55,382 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:55,382 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. files: [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/f921075231ed46e9bc0e168b1ebe29bc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/084ced468747491b85078e09a67a81dc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/257235c3a86f46e681cbff3ba9e8f8c2] 2024-11-11T13:37:55,382 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting f21f7266a43141668d97fdad735eff02, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731332271931 2024-11-11T13:37:55,382 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting f921075231ed46e9bc0e168b1ebe29bc, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731332271931 2024-11-11T13:37:55,382 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting d488a68bb6f243d897ea7cad46e7373b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731332272096 2024-11-11T13:37:55,383 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 084ced468747491b85078e09a67a81dc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731332272096 2024-11-11T13:37:55,383 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 442d4c0c831b4586be093d580c08b5bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731332273299 2024-11-11T13:37:55,383 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] compactions.Compactor(225): Compacting 257235c3a86f46e681cbff3ba9e8f8c2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731332273290 2024-11-11T13:37:55,392 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#B#compaction#555 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:55,392 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/24c39db69ae74af7bebe3a516a1306ca is 50, key is test_row_0/B:col10/1731332273316/Put/seqid=0 2024-11-11T13:37:55,394 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:55,395 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111d3bac051a48345509babc8fa80241a7c_f177ccda479452e8555a646c67251509 store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:55,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742480_1656 (size=13187) 2024-11-11T13:37:55,415 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111d3bac051a48345509babc8fa80241a7c_f177ccda479452e8555a646c67251509, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:55,416 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d3bac051a48345509babc8fa80241a7c_f177ccda479452e8555a646c67251509 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:55,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742481_1657 (size=4469) 2024-11-11T13:37:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36877 {}] regionserver.HRegion(8855): Flush requested on f177ccda479452e8555a646c67251509 2024-11-11T13:37:55,690 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-11T13:37:55,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-11T13:37:55,691 INFO [Thread-2456 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2456. 2024-11-11T13:37:55,691 DEBUG [Thread-2456 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:55,691 DEBUG [Thread-2456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,691 DEBUG [Thread-2456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:55,691 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:55,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111300bd3fbfe5448b496000b25dcb0a2e9_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332274543/Put/seqid=0 2024-11-11T13:37:55,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742482_1658 (size=12454) 2024-11-11T13:37:55,800 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/24c39db69ae74af7bebe3a516a1306ca as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/24c39db69ae74af7bebe3a516a1306ca 2024-11-11T13:37:55,803 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/B of f177ccda479452e8555a646c67251509 into 24c39db69ae74af7bebe3a516a1306ca(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:55,803 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:55,803 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/B, priority=13, startTime=1731332275380; duration=0sec 2024-11-11T13:37:55,804 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T13:37:55,804 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:B 2024-11-11T13:37:55,804 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T13:37:55,804 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T13:37:55,804 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1541): f177ccda479452e8555a646c67251509/C is initiating minor compaction (all files) 2024-11-11T13:37:55,804 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f177ccda479452e8555a646c67251509/C in TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:55,805 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/91aa629eb9134a4a8b0b772008e7249a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/e26d27762b6747b9b0103989c27f2d75, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/320ea84ebd9140ee977b6733362005f3] into tmpdir=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp, totalSize=36.8 K 2024-11-11T13:37:55,805 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 91aa629eb9134a4a8b0b772008e7249a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731332271931 2024-11-11T13:37:55,805 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting e26d27762b6747b9b0103989c27f2d75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731332272096 2024-11-11T13:37:55,805 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] compactions.Compactor(225): Compacting 320ea84ebd9140ee977b6733362005f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731332273299 2024-11-11T13:37:55,812 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#C#compaction#558 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:55,812 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/b707c81145be4fafbb735a58775e5390 is 50, key is test_row_0/C:col10/1731332273316/Put/seqid=0 2024-11-11T13:37:55,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742483_1659 (size=13187) 2024-11-11T13:37:55,820 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f177ccda479452e8555a646c67251509#A#compaction#556 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T13:37:55,820 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/041241bf5137436cbfa3947bc32b899e is 175, key is test_row_0/A:col10/1731332273316/Put/seqid=0 2024-11-11T13:37:55,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742484_1660 (size=32141) 2024-11-11T13:37:56,104 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:56,108 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111300bd3fbfe5448b496000b25dcb0a2e9_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111300bd3fbfe5448b496000b25dcb0a2e9_f177ccda479452e8555a646c67251509 2024-11-11T13:37:56,108 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/5a2ec9ff29f14723a369ebe872a2020a, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:56,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/5a2ec9ff29f14723a369ebe872a2020a is 175, key is test_row_0/A:col10/1731332274543/Put/seqid=0 2024-11-11T13:37:56,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742485_1661 (size=31255) 2024-11-11T13:37:56,220 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/b707c81145be4fafbb735a58775e5390 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/b707c81145be4fafbb735a58775e5390 2024-11-11T13:37:56,226 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/C of f177ccda479452e8555a646c67251509 into b707c81145be4fafbb735a58775e5390(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:56,226 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:56,226 INFO [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/C, priority=13, startTime=1731332275380; duration=0sec 2024-11-11T13:37:56,226 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:56,226 DEBUG [RS:0;37d26ecf6460:36877-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:C 2024-11-11T13:37:56,228 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/041241bf5137436cbfa3947bc32b899e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/041241bf5137436cbfa3947bc32b899e 2024-11-11T13:37:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T13:37:56,229 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:37:56,233 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f177ccda479452e8555a646c67251509/A of f177ccda479452e8555a646c67251509 into 041241bf5137436cbfa3947bc32b899e(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T13:37:56,233 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:56,233 INFO [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., storeName=f177ccda479452e8555a646c67251509/A, priority=13, startTime=1731332275380; duration=0sec 2024-11-11T13:37:56,233 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T13:37:56,233 DEBUG [RS:0;37d26ecf6460:36877-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f177ccda479452e8555a646c67251509:A 2024-11-11T13:37:56,513 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/5a2ec9ff29f14723a369ebe872a2020a 2024-11-11T13:37:56,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/5724c69b0257481bade9c2dac289ad4e is 50, key is test_row_0/B:col10/1731332274543/Put/seqid=0 2024-11-11T13:37:56,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742486_1662 (size=12301) 2024-11-11T13:37:56,851 INFO [Thread-2458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2458. 2024-11-11T13:37:56,851 INFO [Thread-2454 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2454. 2024-11-11T13:37:56,851 DEBUG [Thread-2458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:56,851 DEBUG [Thread-2454 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:56,851 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:56,851 DEBUG [Thread-2454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:56,851 DEBUG [Thread-2454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:56,851 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:56,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:56,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:56,860 INFO [Thread-2460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2460. 2024-11-11T13:37:56,860 INFO [Thread-2452 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2452. 2024-11-11T13:37:56,860 DEBUG [Thread-2452 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:56,860 DEBUG [Thread-2460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T13:37:56,860 DEBUG [Thread-2452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:56,860 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:56,860 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:56,860 DEBUG [Thread-2452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:56,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:56,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 239 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3260 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3348 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3292 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3259 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3307 2024-11-11T13:37:56,861 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T13:37:56,861 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:37:56,861 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:37:56,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:56,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:37:56,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:37:56,862 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T13:37:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T13:37:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T13:37:56,864 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332276864"}]},"ts":"1731332276864"} 2024-11-11T13:37:56,865 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T13:37:56,865 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T13:37:56,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T13:37:56,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, UNASSIGN}] 2024-11-11T13:37:56,867 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, UNASSIGN 2024-11-11T13:37:56,868 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=f177ccda479452e8555a646c67251509, regionState=CLOSING, regionLocation=37d26ecf6460,36877,1731332101576 2024-11-11T13:37:56,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, UNASSIGN because future has completed 2024-11-11T13:37:56,869 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T13:37:56,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576}] 2024-11-11T13:37:56,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/5724c69b0257481bade9c2dac289ad4e 2024-11-11T13:37:56,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/cb8a2dca842d435eaa47b7a4bde2cb9c is 50, key is test_row_0/C:col10/1731332274543/Put/seqid=0 2024-11-11T13:37:56,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742487_1663 (size=12301) 2024-11-11T13:37:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T13:37:57,021 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close f177ccda479452e8555a646c67251509 2024-11-11T13:37:57,021 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T13:37:57,021 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing f177ccda479452e8555a646c67251509, disabling compactions & flushes 2024-11-11T13:37:57,021 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T13:37:57,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/cb8a2dca842d435eaa47b7a4bde2cb9c 2024-11-11T13:37:57,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/5a2ec9ff29f14723a369ebe872a2020a as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/5a2ec9ff29f14723a369ebe872a2020a 2024-11-11T13:37:57,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/5a2ec9ff29f14723a369ebe872a2020a, entries=150, sequenceid=387, filesize=30.5 K 2024-11-11T13:37:57,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/5724c69b0257481bade9c2dac289ad4e as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/5724c69b0257481bade9c2dac289ad4e 2024-11-11T13:37:57,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/5724c69b0257481bade9c2dac289ad4e, entries=150, sequenceid=387, filesize=12.0 K 2024-11-11T13:37:57,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/cb8a2dca842d435eaa47b7a4bde2cb9c as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/cb8a2dca842d435eaa47b7a4bde2cb9c 2024-11-11T13:37:57,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/cb8a2dca842d435eaa47b7a4bde2cb9c, entries=150, sequenceid=387, filesize=12.0 K 2024-11-11T13:37:57,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for f177ccda479452e8555a646c67251509 in 1660ms, sequenceid=387, compaction requested=false 2024-11-11T13:37:57,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f177ccda479452e8555a646c67251509: 2024-11-11T13:37:57,350 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:57,350 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:57,350 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. after waiting 0 ms 2024-11-11T13:37:57,350 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:57,350 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing f177ccda479452e8555a646c67251509 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-11T13:37:57,355 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c4663f536d9749e0ad909a7f933ae1df_f177ccda479452e8555a646c67251509 is 50, key is test_row_0/A:col10/1731332276859/Put/seqid=0 2024-11-11T13:37:57,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742488_1664 (size=12454) 2024-11-11T13:37:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T13:37:57,759 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:37:57,763 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c4663f536d9749e0ad909a7f933ae1df_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c4663f536d9749e0ad909a7f933ae1df_f177ccda479452e8555a646c67251509 2024-11-11T13:37:57,764 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/f720ae68de6f4d06a60a547737566602, store: [table=TestAcidGuarantees family=A region=f177ccda479452e8555a646c67251509] 2024-11-11T13:37:57,764 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/f720ae68de6f4d06a60a547737566602 is 175, key is test_row_0/A:col10/1731332276859/Put/seqid=0 2024-11-11T13:37:57,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742489_1665 (size=31255) 2024-11-11T13:37:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T13:37:58,168 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/f720ae68de6f4d06a60a547737566602 2024-11-11T13:37:58,173 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/77775b7131f142c1a6eedcf96cd9c241 is 50, key is test_row_0/B:col10/1731332276859/Put/seqid=0 2024-11-11T13:37:58,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742490_1666 (size=12301) 2024-11-11T13:37:58,577 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/77775b7131f142c1a6eedcf96cd9c241 2024-11-11T13:37:58,582 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/e7e5675f99674ee6b9c70a518044be6d is 50, key is test_row_0/C:col10/1731332276859/Put/seqid=0 2024-11-11T13:37:58,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742491_1667 (size=12301) 2024-11-11T13:37:58,988 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/e7e5675f99674ee6b9c70a518044be6d 2024-11-11T13:37:58,992 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/A/f720ae68de6f4d06a60a547737566602 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/f720ae68de6f4d06a60a547737566602 2024-11-11T13:37:58,996 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/f720ae68de6f4d06a60a547737566602, entries=150, sequenceid=397, filesize=30.5 K 2024-11-11T13:37:58,996 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/B/77775b7131f142c1a6eedcf96cd9c241 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/77775b7131f142c1a6eedcf96cd9c241 2024-11-11T13:37:59,000 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/77775b7131f142c1a6eedcf96cd9c241, entries=150, sequenceid=397, filesize=12.0 K 2024-11-11T13:37:59,002 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/.tmp/C/e7e5675f99674ee6b9c70a518044be6d as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/e7e5675f99674ee6b9c70a518044be6d 2024-11-11T13:37:59,006 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/e7e5675f99674ee6b9c70a518044be6d, entries=150, sequenceid=397, filesize=12.0 K 2024-11-11T13:37:59,006 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f177ccda479452e8555a646c67251509 in 1656ms, sequenceid=397, compaction requested=true 2024-11-11T13:37:59,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ee57bb59dc8946ad950aa4378062e17d, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b380c4eb34f644c981a0d8db7b6acd30, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/400f65a9ee884414a5ac0731a8b52166, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/18dee53d48ee47e8baa49e3d6d90fbee, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/85fe4c62d2094ff6a7426230dc3027fc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/76b9c572082849f79de34f7e3168b316, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/c01d52731a36433ca7b183afb30aa2e8, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82e875938e53429db48844120ef30a02, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/86827fa657b648b48c6f6189d5181472, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/6bf1b3441ac2499e968338366bbb2e97, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82151367a2bf4716a13028b02d3908b2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/8e095b9e20e449df9cd8f25abd40d1df, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/988ea23465f94cf48d73c270ee41d852, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/77a7b81adc454b0bb143efdbaf4d351c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/301942ce2ee7478d92e430d3741803db, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/e2008215f1904e149f53538283c3a890, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b20377159ab84315850c4f053ea14fc4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/26860d5a2a06480389d18bfcfe11bdbe, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ff9e48997a94448dbb65074b5fd8c1da, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b523a06c5cd8417db753f5e026ce4e1c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/640387c3b18d4f3d8844b99170b11165, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/d4af933c4dca48d798872ebc67d7b2d6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/898931e4f1fc48b8b8271964cd2615f2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/f921075231ed46e9bc0e168b1ebe29bc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/084ced468747491b85078e09a67a81dc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/257235c3a86f46e681cbff3ba9e8f8c2] to archive 2024-11-11T13:37:59,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:37:59,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ee57bb59dc8946ad950aa4378062e17d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ee57bb59dc8946ad950aa4378062e17d 2024-11-11T13:37:59,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b380c4eb34f644c981a0d8db7b6acd30 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b380c4eb34f644c981a0d8db7b6acd30 2024-11-11T13:37:59,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T13:37:59,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/400f65a9ee884414a5ac0731a8b52166 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/400f65a9ee884414a5ac0731a8b52166 2024-11-11T13:37:59,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/18dee53d48ee47e8baa49e3d6d90fbee to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/18dee53d48ee47e8baa49e3d6d90fbee 2024-11-11T13:37:59,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/85fe4c62d2094ff6a7426230dc3027fc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/85fe4c62d2094ff6a7426230dc3027fc 2024-11-11T13:37:59,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/76b9c572082849f79de34f7e3168b316 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/76b9c572082849f79de34f7e3168b316 2024-11-11T13:37:59,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/c01d52731a36433ca7b183afb30aa2e8 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/c01d52731a36433ca7b183afb30aa2e8 2024-11-11T13:37:59,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82e875938e53429db48844120ef30a02 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82e875938e53429db48844120ef30a02 2024-11-11T13:37:59,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/86827fa657b648b48c6f6189d5181472 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/86827fa657b648b48c6f6189d5181472 2024-11-11T13:37:59,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/6bf1b3441ac2499e968338366bbb2e97 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/6bf1b3441ac2499e968338366bbb2e97 2024-11-11T13:37:59,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82151367a2bf4716a13028b02d3908b2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/82151367a2bf4716a13028b02d3908b2 2024-11-11T13:37:59,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/8e095b9e20e449df9cd8f25abd40d1df to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/8e095b9e20e449df9cd8f25abd40d1df 2024-11-11T13:37:59,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/988ea23465f94cf48d73c270ee41d852 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/988ea23465f94cf48d73c270ee41d852 2024-11-11T13:37:59,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/77a7b81adc454b0bb143efdbaf4d351c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/77a7b81adc454b0bb143efdbaf4d351c 2024-11-11T13:37:59,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/301942ce2ee7478d92e430d3741803db to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/301942ce2ee7478d92e430d3741803db 2024-11-11T13:37:59,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/e2008215f1904e149f53538283c3a890 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/e2008215f1904e149f53538283c3a890 2024-11-11T13:37:59,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b20377159ab84315850c4f053ea14fc4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b20377159ab84315850c4f053ea14fc4 2024-11-11T13:37:59,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/26860d5a2a06480389d18bfcfe11bdbe to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/26860d5a2a06480389d18bfcfe11bdbe 2024-11-11T13:37:59,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ff9e48997a94448dbb65074b5fd8c1da to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/ff9e48997a94448dbb65074b5fd8c1da 2024-11-11T13:37:59,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b523a06c5cd8417db753f5e026ce4e1c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/b523a06c5cd8417db753f5e026ce4e1c 2024-11-11T13:37:59,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/640387c3b18d4f3d8844b99170b11165 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/640387c3b18d4f3d8844b99170b11165 2024-11-11T13:37:59,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/d4af933c4dca48d798872ebc67d7b2d6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/d4af933c4dca48d798872ebc67d7b2d6 2024-11-11T13:37:59,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/898931e4f1fc48b8b8271964cd2615f2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/898931e4f1fc48b8b8271964cd2615f2 2024-11-11T13:37:59,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/f921075231ed46e9bc0e168b1ebe29bc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/f921075231ed46e9bc0e168b1ebe29bc 2024-11-11T13:37:59,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/084ced468747491b85078e09a67a81dc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/084ced468747491b85078e09a67a81dc 2024-11-11T13:37:59,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/257235c3a86f46e681cbff3ba9e8f8c2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/257235c3a86f46e681cbff3ba9e8f8c2 2024-11-11T13:37:59,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/227a7dcaaf704c7881ad19079a461be2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/21535d11bbbb4f5ba09d642fbe8527cb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/9c106126e9f64c718d7bf63e6faf2b73, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/c39fdd791fbd450896b03640951888b4, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e017f4029e2d4451bc10200bde6f8dca, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/a1802aa2d79d477cb413d8fa1b5c8db5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e68edb095fbc4122a06a4ed64139a923, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/adf6693a54234bca9d946b17ef78431e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f3110875198f4bab9ba685a164db1b7c, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/ba3812665ce048d1aa2a2a40724360bc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2c84629bb4f848efb2784f741466b1a2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/3f5efb51127e40f0b37ae9f1bff9651e, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/5e1715a79d434d39a5bf187f36e23172, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/813a678fd7d84b31b9a32fee48c7cf37, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2ef29b7fd59f4780a3c2f0db3104abb1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/1740867675b448739ffdba4ae19774fc, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/930bcfb5a46149929942090376ad4832, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/efb21cef4334472281875400c5397a58, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2d3a50a82d93463b909c9391193136a6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f8673c71200b49dea53ef29a19c78da2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d24176c049914cc7a20e7e531cf709de, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/7e49631f23ed48be88be3199fb866e26, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f21f7266a43141668d97fdad735eff02, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/ff6e71266e8f41c09207f5d815e08d80, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d488a68bb6f243d897ea7cad46e7373b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/442d4c0c831b4586be093d580c08b5bf] to archive 2024-11-11T13:37:59,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:37:59,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/227a7dcaaf704c7881ad19079a461be2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/227a7dcaaf704c7881ad19079a461be2 2024-11-11T13:37:59,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/21535d11bbbb4f5ba09d642fbe8527cb to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/21535d11bbbb4f5ba09d642fbe8527cb 2024-11-11T13:37:59,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/9c106126e9f64c718d7bf63e6faf2b73 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/9c106126e9f64c718d7bf63e6faf2b73 2024-11-11T13:37:59,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/c39fdd791fbd450896b03640951888b4 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/c39fdd791fbd450896b03640951888b4 2024-11-11T13:37:59,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e017f4029e2d4451bc10200bde6f8dca to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e017f4029e2d4451bc10200bde6f8dca 2024-11-11T13:37:59,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/a1802aa2d79d477cb413d8fa1b5c8db5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/a1802aa2d79d477cb413d8fa1b5c8db5 2024-11-11T13:37:59,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e68edb095fbc4122a06a4ed64139a923 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/e68edb095fbc4122a06a4ed64139a923 2024-11-11T13:37:59,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/adf6693a54234bca9d946b17ef78431e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/adf6693a54234bca9d946b17ef78431e 2024-11-11T13:37:59,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f3110875198f4bab9ba685a164db1b7c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f3110875198f4bab9ba685a164db1b7c 2024-11-11T13:37:59,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/ba3812665ce048d1aa2a2a40724360bc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/ba3812665ce048d1aa2a2a40724360bc 2024-11-11T13:37:59,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2c84629bb4f848efb2784f741466b1a2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2c84629bb4f848efb2784f741466b1a2 2024-11-11T13:37:59,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/3f5efb51127e40f0b37ae9f1bff9651e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/3f5efb51127e40f0b37ae9f1bff9651e 2024-11-11T13:37:59,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/5e1715a79d434d39a5bf187f36e23172 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/5e1715a79d434d39a5bf187f36e23172 2024-11-11T13:37:59,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/813a678fd7d84b31b9a32fee48c7cf37 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/813a678fd7d84b31b9a32fee48c7cf37 2024-11-11T13:37:59,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2ef29b7fd59f4780a3c2f0db3104abb1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2ef29b7fd59f4780a3c2f0db3104abb1 2024-11-11T13:37:59,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/1740867675b448739ffdba4ae19774fc to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/1740867675b448739ffdba4ae19774fc 2024-11-11T13:37:59,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/930bcfb5a46149929942090376ad4832 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/930bcfb5a46149929942090376ad4832 2024-11-11T13:37:59,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/efb21cef4334472281875400c5397a58 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/efb21cef4334472281875400c5397a58 2024-11-11T13:37:59,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2d3a50a82d93463b909c9391193136a6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/2d3a50a82d93463b909c9391193136a6 2024-11-11T13:37:59,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f8673c71200b49dea53ef29a19c78da2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f8673c71200b49dea53ef29a19c78da2 2024-11-11T13:37:59,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d24176c049914cc7a20e7e531cf709de to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d24176c049914cc7a20e7e531cf709de 2024-11-11T13:37:59,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/7e49631f23ed48be88be3199fb866e26 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/7e49631f23ed48be88be3199fb866e26 2024-11-11T13:37:59,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f21f7266a43141668d97fdad735eff02 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/f21f7266a43141668d97fdad735eff02 2024-11-11T13:37:59,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/ff6e71266e8f41c09207f5d815e08d80 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/ff6e71266e8f41c09207f5d815e08d80 2024-11-11T13:37:59,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d488a68bb6f243d897ea7cad46e7373b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/d488a68bb6f243d897ea7cad46e7373b 2024-11-11T13:37:59,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/442d4c0c831b4586be093d580c08b5bf to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/442d4c0c831b4586be093d580c08b5bf 2024-11-11T13:37:59,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c28d12eaab144924b3866d192817ee45, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/37eda10f8a6845e8809272e397ce4238, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c1925122f9a746a79019b03b55731997, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/d93a64744f394b12bdac2995d28746b5, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/1c6c2d8da39e40e88dbb8adb0e54a946, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/45baaccafa4e447086764bfc6b944b50, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/8b010ab83cb44b38af1e86506f8d5004, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/ffdcf9c1974a43f199a78ce499a7e2db, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c72f34424d7e4eb69a21d9307a6b66f6, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/6e3c98503aeb464692a295c553d3a120, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/f93aec0e17504d5d953f63b186d30b7f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/45ff2bef0b164206827015b1d6d3c5ed, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/11d573baf81646d6a665698d13d044b2, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/503d650d7ed3467cbc289c7195f9dd15, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/9b7c41ac9fae40f1865e4f01bd77fbaa, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/84b7bedebc67455aaad9e0ce70e0d962, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/673e21d897e14dbdb7aa1ad81efd78a1, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/65c22dfb39784f1b9961dde666eacb72, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/d92b1be078e9499298f8ed7e635b03fb, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/4f9a58b69112440fa03623d6840d3aff, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/bb140286afe84fb0b2f8e39b8ae5ae0f, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/020cc4fe6da2486d81364e9163582a8b, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/91aa629eb9134a4a8b0b772008e7249a, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/0afcead517814f39801e03641c762d36, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/e26d27762b6747b9b0103989c27f2d75, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/320ea84ebd9140ee977b6733362005f3] to archive 2024-11-11T13:37:59,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T13:37:59,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c28d12eaab144924b3866d192817ee45 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c28d12eaab144924b3866d192817ee45 2024-11-11T13:37:59,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/37eda10f8a6845e8809272e397ce4238 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/37eda10f8a6845e8809272e397ce4238 2024-11-11T13:37:59,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c1925122f9a746a79019b03b55731997 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c1925122f9a746a79019b03b55731997 2024-11-11T13:37:59,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/d93a64744f394b12bdac2995d28746b5 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/d93a64744f394b12bdac2995d28746b5 2024-11-11T13:37:59,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/1c6c2d8da39e40e88dbb8adb0e54a946 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/1c6c2d8da39e40e88dbb8adb0e54a946 2024-11-11T13:37:59,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/45baaccafa4e447086764bfc6b944b50 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/45baaccafa4e447086764bfc6b944b50 2024-11-11T13:37:59,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/8b010ab83cb44b38af1e86506f8d5004 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/8b010ab83cb44b38af1e86506f8d5004 2024-11-11T13:37:59,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/ffdcf9c1974a43f199a78ce499a7e2db to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/ffdcf9c1974a43f199a78ce499a7e2db 2024-11-11T13:37:59,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c72f34424d7e4eb69a21d9307a6b66f6 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/c72f34424d7e4eb69a21d9307a6b66f6 2024-11-11T13:37:59,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/6e3c98503aeb464692a295c553d3a120 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/6e3c98503aeb464692a295c553d3a120 2024-11-11T13:37:59,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/f93aec0e17504d5d953f63b186d30b7f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/f93aec0e17504d5d953f63b186d30b7f 2024-11-11T13:37:59,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/45ff2bef0b164206827015b1d6d3c5ed to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/45ff2bef0b164206827015b1d6d3c5ed 2024-11-11T13:37:59,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/11d573baf81646d6a665698d13d044b2 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/11d573baf81646d6a665698d13d044b2 2024-11-11T13:37:59,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/503d650d7ed3467cbc289c7195f9dd15 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/503d650d7ed3467cbc289c7195f9dd15 2024-11-11T13:37:59,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/9b7c41ac9fae40f1865e4f01bd77fbaa to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/9b7c41ac9fae40f1865e4f01bd77fbaa 2024-11-11T13:37:59,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/84b7bedebc67455aaad9e0ce70e0d962 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/84b7bedebc67455aaad9e0ce70e0d962 2024-11-11T13:37:59,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/673e21d897e14dbdb7aa1ad81efd78a1 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/673e21d897e14dbdb7aa1ad81efd78a1 2024-11-11T13:37:59,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/65c22dfb39784f1b9961dde666eacb72 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/65c22dfb39784f1b9961dde666eacb72 2024-11-11T13:37:59,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/d92b1be078e9499298f8ed7e635b03fb to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/d92b1be078e9499298f8ed7e635b03fb 2024-11-11T13:37:59,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/4f9a58b69112440fa03623d6840d3aff to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/4f9a58b69112440fa03623d6840d3aff 2024-11-11T13:37:59,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/bb140286afe84fb0b2f8e39b8ae5ae0f to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/bb140286afe84fb0b2f8e39b8ae5ae0f 2024-11-11T13:37:59,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/020cc4fe6da2486d81364e9163582a8b to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/020cc4fe6da2486d81364e9163582a8b 2024-11-11T13:37:59,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/91aa629eb9134a4a8b0b772008e7249a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/91aa629eb9134a4a8b0b772008e7249a 2024-11-11T13:37:59,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/0afcead517814f39801e03641c762d36 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/0afcead517814f39801e03641c762d36 2024-11-11T13:37:59,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/e26d27762b6747b9b0103989c27f2d75 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/e26d27762b6747b9b0103989c27f2d75 2024-11-11T13:37:59,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/320ea84ebd9140ee977b6733362005f3 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/320ea84ebd9140ee977b6733362005f3 2024-11-11T13:37:59,093 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/recovered.edits/400.seqid, newMaxSeqId=400, maxSeqId=4 2024-11-11T13:37:59,094 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. 2024-11-11T13:37:59,094 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for f177ccda479452e8555a646c67251509: Waiting for close lock at 1731332277021Running coprocessor pre-close hooks at 1731332277021Disabling compacts and flushes for region at 1731332277021Disabling writes for close at 1731332277350 (+329 ms)Obtaining lock to block concurrent updates at 1731332277350Preparing flush snapshotting stores in f177ccda479452e8555a646c67251509 at 1731332277350Finished memstore snapshotting TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731332277350Flushing stores of TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509. at 1731332277351 (+1 ms)Flushing f177ccda479452e8555a646c67251509/A: creating writer at 1731332277351Flushing mob file f177ccda479452e8555a646c67251509/A: appending metadata at 1731332277355 (+4 ms)Flushing mob file f177ccda479452e8555a646c67251509/A: closing flushed file at 1731332277355Flushing f177ccda479452e8555a646c67251509/A: appending metadata at 1731332277764 (+409 ms)Flushing f177ccda479452e8555a646c67251509/A: closing flushed file at 1731332277764Flushing f177ccda479452e8555a646c67251509/B: creating writer at 1731332278171 (+407 ms)Flushing f177ccda479452e8555a646c67251509/B: appending metadata at 1731332278173 (+2 ms)Flushing f177ccda479452e8555a646c67251509/B: closing flushed file at 1731332278173Flushing f177ccda479452e8555a646c67251509/C: creating writer at 1731332278580 (+407 ms)Flushing f177ccda479452e8555a646c67251509/C: appending metadata at 1731332278582 (+2 ms)Flushing f177ccda479452e8555a646c67251509/C: closing flushed file at 1731332278582Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60cab856: reopening flushed file at 1731332278992 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@502b0bd8: reopening flushed file at 1731332278996 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dfb2aea: reopening flushed file at 1731332279000 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f177ccda479452e8555a646c67251509 in 1656ms, sequenceid=397, compaction requested=true at 1731332279006 (+6 ms)Writing region close event to WAL at 1731332279090 (+84 ms)Running coprocessor post-close hooks at 1731332279093 (+3 ms)Closed at 1731332279094 (+1 ms) 2024-11-11T13:37:59,096 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed f177ccda479452e8555a646c67251509 2024-11-11T13:37:59,096 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=f177ccda479452e8555a646c67251509, regionState=CLOSED 2024-11-11T13:37:59,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 because future has completed 2024-11-11T13:37:59,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-11T13:37:59,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure f177ccda479452e8555a646c67251509, server=37d26ecf6460,36877,1731332101576 in 2.2300 sec 2024-11-11T13:37:59,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-11T13:37:59,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f177ccda479452e8555a646c67251509, UNASSIGN in 2.2350 sec 2024-11-11T13:37:59,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-11T13:37:59,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2400 sec 2024-11-11T13:37:59,110 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731332279110"}]},"ts":"1731332279110"} 2024-11-11T13:37:59,111 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T13:37:59,112 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T13:37:59,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2500 sec 2024-11-11T13:37:59,757 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T13:38:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T13:38:01,039 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:38:01,040 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T13:38:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:38:01,041 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:38:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T13:38:01,042 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:38:01,044 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,046 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C, FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/recovered.edits] 2024-11-11T13:38:01,047 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/041241bf5137436cbfa3947bc32b899e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/041241bf5137436cbfa3947bc32b899e 2024-11-11T13:38:01,048 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/5a2ec9ff29f14723a369ebe872a2020a to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/5a2ec9ff29f14723a369ebe872a2020a 2024-11-11T13:38:01,049 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/f720ae68de6f4d06a60a547737566602 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/A/f720ae68de6f4d06a60a547737566602 2024-11-11T13:38:01,051 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/24c39db69ae74af7bebe3a516a1306ca to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/24c39db69ae74af7bebe3a516a1306ca 2024-11-11T13:38:01,051 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/5724c69b0257481bade9c2dac289ad4e to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/5724c69b0257481bade9c2dac289ad4e 2024-11-11T13:38:01,052 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/77775b7131f142c1a6eedcf96cd9c241 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/B/77775b7131f142c1a6eedcf96cd9c241 2024-11-11T13:38:01,054 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/b707c81145be4fafbb735a58775e5390 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/b707c81145be4fafbb735a58775e5390 2024-11-11T13:38:01,054 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/cb8a2dca842d435eaa47b7a4bde2cb9c to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/cb8a2dca842d435eaa47b7a4bde2cb9c 2024-11-11T13:38:01,055 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/e7e5675f99674ee6b9c70a518044be6d to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/C/e7e5675f99674ee6b9c70a518044be6d 2024-11-11T13:38:01,058 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/recovered.edits/400.seqid to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509/recovered.edits/400.seqid 2024-11-11T13:38:01,058 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/default/TestAcidGuarantees/f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,058 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T13:38:01,059 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T13:38:01,059 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T13:38:01,061 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111dbb46564cb144e6a27d5a06712ab149_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111dbb46564cb144e6a27d5a06712ab149_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,062 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111287f79e1e625411092fc987439c17d7c_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111287f79e1e625411092fc987439c17d7c_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,063 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111300bd3fbfe5448b496000b25dcb0a2e9_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111300bd3fbfe5448b496000b25dcb0a2e9_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,064 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111132a4720dcd18419a97cd0967cfd709c2_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111132a4720dcd18419a97cd0967cfd709c2_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,065 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111147d26f2df8044aacb1aca2a14ecef70b_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111147d26f2df8044aacb1aca2a14ecef70b_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,066 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b019f1ab51d402eaccf491173a287b8_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b019f1ab51d402eaccf491173a287b8_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,066 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111566beefa6aa84370bb7c9c792e233dcb_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111566beefa6aa84370bb7c9c792e233dcb_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,067 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111593992fe433d4689a4a6a95280d6696e_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111593992fe433d4689a4a6a95280d6696e_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,068 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115e59e7d66e57483caa7f6db7e2b53c99_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115e59e7d66e57483caa7f6db7e2b53c99_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,069 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111161ec6b619898426d95fed473030a7291_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111161ec6b619898426d95fed473030a7291_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,069 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111655c6513b3c141509aac089d34a237e4_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111655c6513b3c141509aac089d34a237e4_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,070 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111167448763db1f4d85bc1cdbb0c744003c_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111167448763db1f4d85bc1cdbb0c744003c_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,071 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170f174de9b0949aa9695eb26529c4841_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170f174de9b0949aa9695eb26529c4841_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,072 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111195dab5ed3d2b4535b8c4d38d9e8cc380_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111195dab5ed3d2b4535b8c4d38d9e8cc380_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,073 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111963800a9d1644498bb42a10d48acd9f0_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111963800a9d1644498bb42a10d48acd9f0_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,074 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b010ca279fd24aa7a7759e1151e400e5_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b010ca279fd24aa7a7759e1151e400e5_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,074 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b509a8c7f97a422b8ce1a70897acda4e_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b509a8c7f97a422b8ce1a70897acda4e_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,075 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b55d1bbb8e754dfbb19b290d607d677f_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b55d1bbb8e754dfbb19b290d607d677f_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,076 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b7d6b5d089604bcd8b711ab7bca1bd92_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b7d6b5d089604bcd8b711ab7bca1bd92_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,077 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bf7ab6c91b3540949e0eaee306b5cacc_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bf7ab6c91b3540949e0eaee306b5cacc_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,078 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c4663f536d9749e0ad909a7f933ae1df_f177ccda479452e8555a646c67251509 to hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c4663f536d9749e0ad909a7f933ae1df_f177ccda479452e8555a646c67251509 2024-11-11T13:38:01,078 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T13:38:01,080 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:38:01,082 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T13:38:01,083 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T13:38:01,084 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:38:01,084 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T13:38:01,084 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731332281084"}]},"ts":"9223372036854775807"} 2024-11-11T13:38:01,086 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T13:38:01,086 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f177ccda479452e8555a646c67251509, NAME => 'TestAcidGuarantees,,1731332251876.f177ccda479452e8555a646c67251509.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T13:38:01,086 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T13:38:01,086 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731332281086"}]},"ts":"9223372036854775807"} 2024-11-11T13:38:01,087 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T13:38:01,088 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T13:38:01,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-11T13:38:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35609 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T13:38:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T13:38:01,150 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T13:38:01,161 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T13:38:01,163 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 239) - Thread LEAK? -, OpenFileDescriptor=455 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=595 (was 662), ProcessCount=11 (was 11), AvailableMemoryMB=2489 (was 2588) 2024-11-11T13:38:01,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T13:38:01,163 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:38:01,163 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:38:01,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:38:01,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:38:01,163 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:38:01,163 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T13:38:01,163 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=855483586, stopped=false 2024-11-11T13:38:01,164 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=37d26ecf6460,35609,1731332100707 2024-11-11T13:38:01,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T13:38:01,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T13:38:01,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:38:01,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:38:01,269 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T13:38:01,270 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:38:01,270 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:38:01,270 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:38:01,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:38:01,271 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '37d26ecf6460,36877,1731332101576' ***** 2024-11-11T13:38:01,271 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T13:38:01,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:38:01,271 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T13:38:01,271 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T13:38:01,272 INFO [RS:0;37d26ecf6460:36877 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T13:38:01,272 INFO [RS:0;37d26ecf6460:36877 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T13:38:01,272 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(959): stopping server 37d26ecf6460,36877,1731332101576 2024-11-11T13:38:01,272 INFO [RS:0;37d26ecf6460:36877 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T13:38:01,272 INFO [RS:0;37d26ecf6460:36877 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;37d26ecf6460:36877. 2024-11-11T13:38:01,272 DEBUG [RS:0;37d26ecf6460:36877 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:38:01,272 DEBUG [RS:0;37d26ecf6460:36877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:38:01,272 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T13:38:01,273 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T13:38:01,273 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T13:38:01,273 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T13:38:01,273 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T13:38:01,273 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T13:38:01,274 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T13:38:01,274 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T13:38:01,274 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T13:38:01,274 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T13:38:01,274 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T13:38:01,274 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T13:38:01,274 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T13:38:01,278 INFO [regionserver/37d26ecf6460:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:38:01,300 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/info/602d032793d04553bda3b59ef9e51770 is 95, key is TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436./info:/1731332133260/DeleteFamily/seqid=0 2024-11-11T13:38:01,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742492_1668 (size=6514) 2024-11-11T13:38:01,474 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T13:38:01,674 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T13:38:01,707 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/info/602d032793d04553bda3b59ef9e51770 2024-11-11T13:38:01,735 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/ns/17eefda2b78b42eda507fdef826304a5 is 93, key is TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436./ns:/1731332133225/DeleteFamily/seqid=0 2024-11-11T13:38:01,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742493_1669 (size=5906) 2024-11-11T13:38:01,799 INFO [regionserver/37d26ecf6460:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T13:38:01,799 INFO [regionserver/37d26ecf6460:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T13:38:01,874 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T13:38:02,075 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T13:38:02,145 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/ns/17eefda2b78b42eda507fdef826304a5 2024-11-11T13:38:02,164 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/rep_barrier/b7cefe584c9c4e31a3bf208f007c6fed is 102, key is TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436./rep_barrier:/1731332133225/DeleteFamily/seqid=0 2024-11-11T13:38:02,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742494_1670 (size=6025) 2024-11-11T13:38:02,275 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T13:38:02,475 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T13:38:02,569 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/rep_barrier/b7cefe584c9c4e31a3bf208f007c6fed 2024-11-11T13:38:02,596 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/table/edb6616fc58e4e4e8c239e01b20ef444 is 96, key is TestAcidGuarantees,,1731332105386.3b71678035107ffe2dbde3bb5bfff436./table:/1731332133225/DeleteFamily/seqid=0 2024-11-11T13:38:02,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742495_1671 (size=5936) 2024-11-11T13:38:02,675 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T13:38:02,875 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T13:38:03,004 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/table/edb6616fc58e4e4e8c239e01b20ef444 2024-11-11T13:38:03,009 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/info/602d032793d04553bda3b59ef9e51770 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/info/602d032793d04553bda3b59ef9e51770 2024-11-11T13:38:03,013 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/info/602d032793d04553bda3b59ef9e51770, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T13:38:03,015 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/ns/17eefda2b78b42eda507fdef826304a5 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/ns/17eefda2b78b42eda507fdef826304a5 2024-11-11T13:38:03,019 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/ns/17eefda2b78b42eda507fdef826304a5, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T13:38:03,020 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/rep_barrier/b7cefe584c9c4e31a3bf208f007c6fed as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/rep_barrier/b7cefe584c9c4e31a3bf208f007c6fed 2024-11-11T13:38:03,024 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/rep_barrier/b7cefe584c9c4e31a3bf208f007c6fed, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T13:38:03,025 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/.tmp/table/edb6616fc58e4e4e8c239e01b20ef444 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/table/edb6616fc58e4e4e8c239e01b20ef444 2024-11-11T13:38:03,028 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/table/edb6616fc58e4e4e8c239e01b20ef444, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T13:38:03,029 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1755ms, sequenceid=90, compaction requested=false 2024-11-11T13:38:03,034 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T13:38:03,035 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T13:38:03,035 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T13:38:03,035 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731332281273Running coprocessor pre-close hooks at 1731332281274 (+1 ms)Disabling compacts and flushes for region at 1731332281274Disabling writes for close at 1731332281274Obtaining lock to block concurrent updates at 1731332281274Preparing flush snapshotting stores in 1588230740 at 1731332281274Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731332281274Flushing stores of hbase:meta,,1.1588230740 at 1731332281275 (+1 ms)Flushing 1588230740/info: creating writer at 1731332281275Flushing 1588230740/info: appending metadata at 1731332281300 (+25 ms)Flushing 1588230740/info: closing flushed file at 1731332281300Flushing 1588230740/ns: creating writer at 1731332281711 (+411 ms)Flushing 1588230740/ns: appending metadata at 1731332281735 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1731332281735Flushing 1588230740/rep_barrier: creating writer at 1731332282149 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731332282164 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731332282164Flushing 1588230740/table: creating writer at 1731332282572 (+408 ms)Flushing 1588230740/table: appending metadata at 1731332282595 (+23 ms)Flushing 1588230740/table: closing flushed file at 1731332282595Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36c78617: reopening flushed file at 1731332283008 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5037f859: reopening flushed file at 1731332283014 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6493bf5: reopening flushed file at 1731332283019 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2985ea53: reopening flushed file at 1731332283024 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1755ms, sequenceid=90, compaction requested=false at 1731332283029 (+5 ms)Writing region close event to WAL at 1731332283032 (+3 ms)Running coprocessor post-close hooks at 1731332283035 (+3 ms)Closed at 1731332283035 2024-11-11T13:38:03,035 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T13:38:03,076 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(976): stopping server 37d26ecf6460,36877,1731332101576; all regions closed. 2024-11-11T13:38:03,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741834_1010 (size=25344) 2024-11-11T13:38:03,084 DEBUG [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/oldWALs 2024-11-11T13:38:03,084 INFO [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37d26ecf6460%2C36877%2C1731332101576.meta:.meta(num 1731332104921) 2024-11-11T13:38:03,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741832_1008 (size=15522165) 2024-11-11T13:38:03,089 DEBUG [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/oldWALs 2024-11-11T13:38:03,089 INFO [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37d26ecf6460%2C36877%2C1731332101576:(num 1731332103962) 2024-11-11T13:38:03,089 DEBUG [RS:0;37d26ecf6460:36877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:38:03,089 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:38:03,089 INFO [RS:0;37d26ecf6460:36877 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T13:38:03,089 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(370): Chore service for: regionserver/37d26ecf6460:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T13:38:03,090 INFO [RS:0;37d26ecf6460:36877 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T13:38:03,090 INFO [regionserver/37d26ecf6460:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T13:38:03,090 INFO [RS:0;37d26ecf6460:36877 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36877 2024-11-11T13:38:03,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/37d26ecf6460,36877,1731332101576 2024-11-11T13:38:03,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T13:38:03,119 INFO [RS:0;37d26ecf6460:36877 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T13:38:03,128 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [37d26ecf6460,36877,1731332101576] 2024-11-11T13:38:03,136 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/37d26ecf6460,36877,1731332101576 already deleted, retry=false 2024-11-11T13:38:03,136 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 37d26ecf6460,36877,1731332101576 expired; onlineServers=0 2024-11-11T13:38:03,136 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '37d26ecf6460,35609,1731332100707' ***** 2024-11-11T13:38:03,136 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T13:38:03,136 INFO [M:0;37d26ecf6460:35609 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T13:38:03,137 INFO [M:0;37d26ecf6460:35609 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T13:38:03,137 DEBUG [M:0;37d26ecf6460:35609 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T13:38:03,137 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T13:38:03,137 DEBUG [M:0;37d26ecf6460:35609 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T13:38:03,137 DEBUG [master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.small.0-1731332103583 {}] cleaner.HFileCleaner(306): Exit Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.small.0-1731332103583,5,FailOnTimeoutGroup] 2024-11-11T13:38:03,137 DEBUG [master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.large.0-1731332103579 {}] cleaner.HFileCleaner(306): Exit Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.large.0-1731332103579,5,FailOnTimeoutGroup] 2024-11-11T13:38:03,137 INFO [M:0;37d26ecf6460:35609 {}] hbase.ChoreService(370): Chore service for: master/37d26ecf6460:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T13:38:03,137 INFO [M:0;37d26ecf6460:35609 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T13:38:03,137 DEBUG [M:0;37d26ecf6460:35609 {}] master.HMaster(1795): Stopping service threads 2024-11-11T13:38:03,137 INFO [M:0;37d26ecf6460:35609 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T13:38:03,137 INFO [M:0;37d26ecf6460:35609 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T13:38:03,137 ERROR [M:0;37d26ecf6460:35609 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T13:38:03,138 INFO [M:0;37d26ecf6460:35609 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T13:38:03,138 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T13:38:03,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T13:38:03,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:38:03,145 DEBUG [M:0;37d26ecf6460:35609 {}] zookeeper.ZKUtil(347): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T13:38:03,145 WARN [M:0;37d26ecf6460:35609 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T13:38:03,145 INFO [M:0;37d26ecf6460:35609 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/.lastflushedseqids 2024-11-11T13:38:03,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742496_1672 (size=72) 2024-11-11T13:38:03,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:38:03,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b6500d60001, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:38:03,229 INFO [RS:0;37d26ecf6460:36877 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T13:38:03,229 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1031): Exiting; stopping=37d26ecf6460,36877,1731332101576; zookeeper connection closed. 2024-11-11T13:38:03,230 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@83892a7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@83892a7 2024-11-11T13:38:03,230 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T13:38:03,554 INFO [M:0;37d26ecf6460:35609 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T13:38:03,554 INFO [M:0;37d26ecf6460:35609 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T13:38:03,554 DEBUG [M:0;37d26ecf6460:35609 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T13:38:03,554 INFO [M:0;37d26ecf6460:35609 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:38:03,554 DEBUG [M:0;37d26ecf6460:35609 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:38:03,554 DEBUG [M:0;37d26ecf6460:35609 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T13:38:03,554 DEBUG [M:0;37d26ecf6460:35609 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:38:03,554 INFO [M:0;37d26ecf6460:35609 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=767.08 KB heapSize=943.59 KB 2024-11-11T13:38:03,578 DEBUG [M:0;37d26ecf6460:35609 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4f198565b2be462e92a2124caaee3547 is 82, key is hbase:meta,,1/info:regioninfo/1731332105021/Put/seqid=0 2024-11-11T13:38:03,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742497_1673 (size=5672) 2024-11-11T13:38:03,983 INFO [M:0;37d26ecf6460:35609 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2187 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4f198565b2be462e92a2124caaee3547 2024-11-11T13:38:04,009 DEBUG [M:0;37d26ecf6460:35609 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1cc76c818bda4961a69b97761b2617f0 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x8B/proc:d/1731332254949/Put/seqid=0 2024-11-11T13:38:04,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742498_1674 (size=42807) 2024-11-11T13:38:04,417 INFO [M:0;37d26ecf6460:35609 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=766.52 KB at sequenceid=2187 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1cc76c818bda4961a69b97761b2617f0 2024-11-11T13:38:04,421 INFO [M:0;37d26ecf6460:35609 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1cc76c818bda4961a69b97761b2617f0 2024-11-11T13:38:04,438 DEBUG [M:0;37d26ecf6460:35609 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/21f4b098f6394e70b418eff343dc21e0 is 69, key is 37d26ecf6460,36877,1731332101576/rs:state/1731332103694/Put/seqid=0 2024-11-11T13:38:04,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073742499_1675 (size=5156) 2024-11-11T13:38:04,611 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T13:38:04,844 INFO [M:0;37d26ecf6460:35609 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2187 (bloomFilter=true), to=hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/21f4b098f6394e70b418eff343dc21e0 2024-11-11T13:38:04,847 DEBUG [M:0;37d26ecf6460:35609 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4f198565b2be462e92a2124caaee3547 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4f198565b2be462e92a2124caaee3547 2024-11-11T13:38:04,850 INFO [M:0;37d26ecf6460:35609 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4f198565b2be462e92a2124caaee3547, entries=8, sequenceid=2187, filesize=5.5 K 2024-11-11T13:38:04,851 DEBUG [M:0;37d26ecf6460:35609 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1cc76c818bda4961a69b97761b2617f0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1cc76c818bda4961a69b97761b2617f0 2024-11-11T13:38:04,853 INFO [M:0;37d26ecf6460:35609 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1cc76c818bda4961a69b97761b2617f0 2024-11-11T13:38:04,853 INFO [M:0;37d26ecf6460:35609 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1cc76c818bda4961a69b97761b2617f0, entries=162, sequenceid=2187, filesize=41.8 K 2024-11-11T13:38:04,854 DEBUG [M:0;37d26ecf6460:35609 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/21f4b098f6394e70b418eff343dc21e0 as hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/21f4b098f6394e70b418eff343dc21e0 2024-11-11T13:38:04,856 INFO [M:0;37d26ecf6460:35609 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36899/user/jenkins/test-data/c8c41d61-d1f9-7fa6-4c30-071be232bd92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/21f4b098f6394e70b418eff343dc21e0, entries=1, sequenceid=2187, filesize=5.0 K 2024-11-11T13:38:04,857 INFO [M:0;37d26ecf6460:35609 {}] regionserver.HRegion(3140): Finished flush of dataSize ~767.08 KB/785490, heapSize ~943.29 KB/965928, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1303ms, sequenceid=2187, compaction requested=false 2024-11-11T13:38:04,858 INFO [M:0;37d26ecf6460:35609 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:38:04,858 DEBUG [M:0;37d26ecf6460:35609 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731332283554Disabling compacts and flushes for region at 1731332283554Disabling writes for close at 1731332283554Obtaining lock to block concurrent updates at 1731332283554Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731332283554Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=785490, getHeapSize=966168, getOffHeapSize=0, getCellsCount=2374 at 1731332283555 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731332283555Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731332283555Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731332283578 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731332283578Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731332283987 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731332284008 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731332284008Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731332284421 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731332284437 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731332284437Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b454809: reopening flushed file at 1731332284847 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bb75b84: reopening flushed file at 1731332284850 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@689d518: reopening flushed file at 1731332284854 (+4 ms)Finished flush of dataSize ~767.08 KB/785490, heapSize ~943.29 KB/965928, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1303ms, sequenceid=2187, compaction requested=false at 1731332284857 (+3 ms)Writing region close event to WAL at 1731332284858 (+1 ms)Closed at 1731332284858 2024-11-11T13:38:04,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46037 is added to blk_1073741830_1006 (size=928812) 2024-11-11T13:38:04,861 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T13:38:04,861 INFO [M:0;37d26ecf6460:35609 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T13:38:04,861 INFO [M:0;37d26ecf6460:35609 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35609 2024-11-11T13:38:04,861 INFO [M:0;37d26ecf6460:35609 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T13:38:05,036 INFO [M:0;37d26ecf6460:35609 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T13:38:05,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:38:05,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35609-0x1012b6500d60000, quorum=127.0.0.1:52686, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:38:05,045 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@58e0e813{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T13:38:05,048 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@716e8981{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T13:38:05,049 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T13:38:05,049 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@50fd64ad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T13:38:05,049 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d058786{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/hadoop.log.dir/,STOPPED} 2024-11-11T13:38:05,052 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T13:38:05,052 WARN [BP-476254492-172.17.0.3-1731332096997 heartbeating to localhost/127.0.0.1:36899 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T13:38:05,052 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T13:38:05,052 WARN [BP-476254492-172.17.0.3-1731332096997 heartbeating to localhost/127.0.0.1:36899 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-476254492-172.17.0.3-1731332096997 (Datanode Uuid 077efd71-23ba-4051-bb1b-fe9a3773ce51) service to localhost/127.0.0.1:36899 2024-11-11T13:38:05,055 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/cluster_53e0cb5c-a2a6-7eb1-6f1f-43fb1df4ddb1/data/data1/current/BP-476254492-172.17.0.3-1731332096997 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T13:38:05,056 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/cluster_53e0cb5c-a2a6-7eb1-6f1f-43fb1df4ddb1/data/data2/current/BP-476254492-172.17.0.3-1731332096997 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T13:38:05,056 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T13:38:05,068 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a5ffc6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T13:38:05,069 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f30a77e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T13:38:05,069 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T13:38:05,069 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40029c9a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T13:38:05,069 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f9fd429{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/187ecc59-bcb7-277c-671f-f0055c608813/hadoop.log.dir/,STOPPED} 2024-11-11T13:38:05,094 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T13:38:05,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down